[ Mar 22 20:02:51 Enabled. ]
[ Mar 22 20:02:51 Rereading configuration. ]
[ Mar 22 20:02:53 Rereading configuration. ]
[ Mar 22 20:02:57 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_028f1fde-677e-4cf5-82ae-03cd4acf2d32/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_028f1fde-677e-4cf5-82ae-03cd4acf2d32/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:02:57.896864688Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:02:57.900081496Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:02:57.900091746Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:02:57.916538145Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:02:58.052902412Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:23.874701277Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:63865"}
{"msg":"region 153840b8-c0e7-4bff-b1af-5edb6195dda4 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:23.935715973Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:23.940973405Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"uri":"/crucible/0/regions","method":"POST","req_id":"7aec95d3-e2e3-4648-883e-268de272e8d7","remote_addr":"[fd00:1122:3344:101::b]:63865","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":29391,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:23.943689903Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_028f1fde-677e-4cf5-82ae-03cd4acf2d32/crucible/regions/153840b8-c0e7-4bff-b1af-5edb6195dda4","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:23.957322872Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_028f1fde-677e-4cf5-82ae-03cd4acf2d32/crucible/regions/153840b8-c0e7-4bff-b1af-5edb6195dda4","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:23.957356353Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"153840b8-c0e7-4bff-b1af-5edb6195dda4\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/153840b8-c0e7-4bff-b1af-5edb6195dda4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:24.008021969Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"region":"153840b8-c0e7-4bff-b1af-5edb6195dda4","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"153840b8-c0e7-4bff-b1af-5edb6195dda4\" \"--data\" \"/data/regions/153840b8-c0e7-4bff-b1af-5edb6195dda4\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:24.010183945Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"region":"153840b8-c0e7-4bff-b1af-5edb6195dda4","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:24.131947701Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"uri":"/crucible/0/regions","method":"POST","req_id":"004b8436-a075-404d-962c-b32bb03a94d3","remote_addr":"[fd00:1122:3344:101::b]:63865","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":79,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:24.307225994Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"region":"153840b8-c0e7-4bff-b1af-5edb6195dda4","component":"worker"}
{"msg":"region 153840b8-c0e7-4bff-b1af-5edb6195dda4 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:24.307270355Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:24.307408728Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-153840b8-c0e7-4bff-b1af-5edb6195dda4","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:24.308100403Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-153840b8-c0e7-4bff-b1af-5edb6195dda4","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:24.31314473Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:24.323607612Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-153840b8-c0e7-4bff-b1af-5edb6195dda4","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:24.326241158Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/153840b8-c0e7-4bff-b1af-5edb6195dda4","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:24.326979864Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:24.327044435Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:24.327086176Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:24.327125257Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:24.330775554Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:24.341659875Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:24.835841891Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"uri":"/crucible/0/regions","method":"POST","req_id":"f1124a62-9614-4382-bee6-236209a180b4","remote_addr":"[fd00:1122:3344:101::b]:63865","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":577,"response_code":"200"}
{"msg":"region 880d48ca-311c-4e5f-a802-646ba9a892e1 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:29.960170667Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:29.960364342Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"uri":"/crucible/0/regions","method":"POST","req_id":"f294a67c-a01a-458b-9456-8db4101ae3a4","remote_addr":"[fd00:1122:3344:101::b]:63865","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":238,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:29.960379672Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_028f1fde-677e-4cf5-82ae-03cd4acf2d32/crucible/regions/880d48ca-311c-4e5f-a802-646ba9a892e1","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:29.969533506Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_028f1fde-677e-4cf5-82ae-03cd4acf2d32/crucible/regions/880d48ca-311c-4e5f-a802-646ba9a892e1","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:29.969567097Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"880d48ca-311c-4e5f-a802-646ba9a892e1\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/880d48ca-311c-4e5f-a802-646ba9a892e1\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:30.017383162Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"region":"880d48ca-311c-4e5f-a802-646ba9a892e1","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"880d48ca-311c-4e5f-a802-646ba9a892e1\" \"--data\" \"/data/regions/880d48ca-311c-4e5f-a802-646ba9a892e1\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:30.017439544Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"region":"880d48ca-311c-4e5f-a802-646ba9a892e1","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:30.092346455Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"region":"880d48ca-311c-4e5f-a802-646ba9a892e1","component":"worker"}
{"msg":"region 880d48ca-311c-4e5f-a802-646ba9a892e1 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:30.092402546Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:30.092489748Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-880d48ca-311c-4e5f-a802-646ba9a892e1","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:30.094107192Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-880d48ca-311c-4e5f-a802-646ba9a892e1","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:30.096943982Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:30.104471432Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-880d48ca-311c-4e5f-a802-646ba9a892e1","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:30.107358684Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/880d48ca-311c-4e5f-a802-646ba9a892e1","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:30.10769044Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:30.107762682Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:30.107816523Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:30.107873744Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:30.111571523Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:30.115748272Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:30.162174648Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"uri":"/crucible/0/regions","method":"POST","req_id":"b5299509-daf4-41fd-9a06-1dbfc57714eb","remote_addr":"[fd00:1122:3344:101::b]:63865","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:55.546295052Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"uri":"/crucible/0/regions/880d48ca-311c-4e5f-a802-646ba9a892e1","method":"GET","req_id":"bbc51afd-9733-4c2b-8489-8d1b6d638532","remote_addr":"[fd00:1122:3344:101::b]:63865","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":2088,"response_code":"200"}
{"msg":"path is \"/data/regions/880d48ca-311c-4e5f-a802-646ba9a892e1\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:55.54993928Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"datafile"}
{"msg":"dataset is oxp_028f1fde-677e-4cf5-82ae-03cd4acf2d32/crucible/regions/880d48ca-311c-4e5f-a802-646ba9a892e1","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:55.561394163Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"datafile"}
{"msg":"region 880d48ca-311c-4e5f-a802-646ba9a892e1 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:55.570861804Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:55.571076109Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:55.57112405Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"uri":"/crucible/0/regions/880d48ca-311c-4e5f-a802-646ba9a892e1","method":"DELETE","req_id":"65d82c7f-ec18-4cfb-8f35-0698341a0485","remote_addr":"[fd00:1122:3344:101::b]:63865","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":21208,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-880d48ca-311c-4e5f-a802-646ba9a892e1 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:55.572187452Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:55.574147684Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"uri":"/crucible/0/regions/880d48ca-311c-4e5f-a802-646ba9a892e1","method":"GET","req_id":"a0cfef9a-a879-43f6-967f-1f31adeb8e54","remote_addr":"[fd00:1122:3344:101::b]:63865","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":26,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:55.577690979Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_028f1fde-677e-4cf5-82ae-03cd4acf2d32/crucible/regions/880d48ca-311c-4e5f-a802-646ba9a892e1\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:55.58998734Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"region":"880d48ca-311c-4e5f-a802-646ba9a892e1","component":"worker"}
{"msg":"region 880d48ca-311c-4e5f-a802-646ba9a892e1 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:55.642151098Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-22T20:14:55.65354819Z","hostname":"oxz_crucible_0363940a-5514-4a36-88db-915374815563","pid":21794,"uri":"/crucible/0/regions/880d48ca-311c-4e5f-a802-646ba9a892e1","method":"GET","req_id":"c56e6866-970d-4f32-8c42-fd5e706d066c","remote_addr":"[fd00:1122:3344:101::b]:63865","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":58,"response_code":"200"}