[ Mar  2 04:55:39 Enabled. ]
[ Mar  2 04:55:39 Rereading configuration. ]
[ Mar  2 04:55:42 Rereading configuration. ]
[ Mar  2 04:55:47 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_02f51c32-d098-4db1-aebc-508861ba7b69/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_02f51c32-d098-4db1-aebc-508861ba7b69/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:55:47.88641001Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:55:47.890455893Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:55:47.890467513Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:55:47.912240386Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:55:48.071483601Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:30.355408627Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:65371"}
{"msg":"region 6e6a6fad-2c70-4305-b0d4-317b4227ee2f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:30.391603898Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:30.394723179Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"uri":"/crucible/0/regions","method":"POST","req_id":"2453c6e1-e932-4eb2-ab6d-646a3b17ff30","remote_addr":"[fd00:1122:3344:101::c]:65371","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":17730,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:30.394742409Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_02f51c32-d098-4db1-aebc-508861ba7b69/crucible/regions/6e6a6fad-2c70-4305-b0d4-317b4227ee2f","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:30.403898528Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_02f51c32-d098-4db1-aebc-508861ba7b69/crucible/regions/6e6a6fad-2c70-4305-b0d4-317b4227ee2f","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:30.403930249Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"6e6a6fad-2c70-4305-b0d4-317b4227ee2f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6e6a6fad-2c70-4305-b0d4-317b4227ee2f\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:30.47619001Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"region":"6e6a6fad-2c70-4305-b0d4-317b4227ee2f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6e6a6fad-2c70-4305-b0d4-317b4227ee2f\" \"--data\" \"/data/regions/6e6a6fad-2c70-4305-b0d4-317b4227ee2f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:30.481846664Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"region":"6e6a6fad-2c70-4305-b0d4-317b4227ee2f","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:30.621032926Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"region":"6e6a6fad-2c70-4305-b0d4-317b4227ee2f","component":"worker"}
{"msg":"region 6e6a6fad-2c70-4305-b0d4-317b4227ee2f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:30.621083707Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:30.621164898Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-6e6a6fad-2c70-4305-b0d4-317b4227ee2f","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:30.621554033Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6e6a6fad-2c70-4305-b0d4-317b4227ee2f","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:30.625879209Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:30.635377243Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6e6a6fad-2c70-4305-b0d4-317b4227ee2f","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:30.638526854Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6e6a6fad-2c70-4305-b0d4-317b4227ee2f","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:30.639424056Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:30.639502777Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:30.639555847Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:30.639602588Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:30.644136797Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:30.64823026Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:30.757436763Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"uri":"/crucible/0/regions","method":"POST","req_id":"f6519c2e-73bf-4ba6-901a-7950eaab430f","remote_addr":"[fd00:1122:3344:101::c]:65371","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":63,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:31.532787122Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"uri":"/crucible/0/regions/6e6a6fad-2c70-4305-b0d4-317b4227ee2f","method":"GET","req_id":"b452fcf6-9b6e-4661-ac55-1d7ddd4ba721","remote_addr":"[fd00:1122:3344:101::c]:65371","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":2308,"response_code":"200"}
{"msg":"path is \"/data/regions/6e6a6fad-2c70-4305-b0d4-317b4227ee2f\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:31.535632939Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"datafile"}
{"msg":"dataset is oxp_02f51c32-d098-4db1-aebc-508861ba7b69/crucible/regions/6e6a6fad-2c70-4305-b0d4-317b4227ee2f","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:31.547285631Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"datafile"}
{"msg":"region 6e6a6fad-2c70-4305-b0d4-317b4227ee2f state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:31.559126795Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:31.559325068Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:31.559403759Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"uri":"/crucible/0/regions/6e6a6fad-2c70-4305-b0d4-317b4227ee2f","method":"DELETE","req_id":"ebbce18a-839f-4af2-8fdb-de3cfc02707d","remote_addr":"[fd00:1122:3344:101::c]:65371","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":23805,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-6e6a6fad-2c70-4305-b0d4-317b4227ee2f (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:31.560576354Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:31.563050817Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"uri":"/crucible/0/regions/6e6a6fad-2c70-4305-b0d4-317b4227ee2f","method":"GET","req_id":"39dc664b-588b-4e04-8782-573b484ec385","remote_addr":"[fd00:1122:3344:101::c]:65371","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":35,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:31.565470738Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_02f51c32-d098-4db1-aebc-508861ba7b69/crucible/regions/6e6a6fad-2c70-4305-b0d4-317b4227ee2f\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:31.577492385Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"region":"6e6a6fad-2c70-4305-b0d4-317b4227ee2f","component":"worker"}
{"msg":"region 6e6a6fad-2c70-4305-b0d4-317b4227ee2f state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:31.643252711Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:31.706755898Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"uri":"/crucible/0/regions/6e6a6fad-2c70-4305-b0d4-317b4227ee2f","method":"GET","req_id":"24de6c42-b944-4268-a18f-c1c749d0cdc1","remote_addr":"[fd00:1122:3344:101::c]:65371","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":92,"response_code":"200"}
{"msg":"region 83855937-1ef9-45cf-b943-5c66e8ae8259 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:34.060991014Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:34.061142256Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:34.061152216Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"uri":"/crucible/0/regions","method":"POST","req_id":"19cb1330-e00e-40b1-b7a0-e788c5ef44de","remote_addr":"[fd00:1122:3344:101::c]:65371","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":220,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_02f51c32-d098-4db1-aebc-508861ba7b69/crucible/regions/83855937-1ef9-45cf-b943-5c66e8ae8259","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:34.070513838Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_02f51c32-d098-4db1-aebc-508861ba7b69/crucible/regions/83855937-1ef9-45cf-b943-5c66e8ae8259","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:34.070549408Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"83855937-1ef9-45cf-b943-5c66e8ae8259\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/83855937-1ef9-45cf-b943-5c66e8ae8259\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:34.124650413Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"region":"83855937-1ef9-45cf-b943-5c66e8ae8259","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"83855937-1ef9-45cf-b943-5c66e8ae8259\" \"--data\" \"/data/regions/83855937-1ef9-45cf-b943-5c66e8ae8259\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:34.124695724Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"region":"83855937-1ef9-45cf-b943-5c66e8ae8259","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:34.197725395Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"region":"83855937-1ef9-45cf-b943-5c66e8ae8259","component":"worker"}
{"msg":"region 83855937-1ef9-45cf-b943-5c66e8ae8259 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:34.197767135Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:34.197855257Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-83855937-1ef9-45cf-b943-5c66e8ae8259","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:34.199204554Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-83855937-1ef9-45cf-b943-5c66e8ae8259","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:34.201801938Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:34.207847937Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-83855937-1ef9-45cf-b943-5c66e8ae8259","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:34.210482651Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/83855937-1ef9-45cf-b943-5c66e8ae8259","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:34.210706674Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:34.210754205Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:34.210793705Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:34.210829546Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:34.213952166Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:34.218146491Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-02T04:56:34.272247386Z","hostname":"oxz_crucible_9ae2af37-6165-4f8f-8100-b19e0f9ec833","pid":21887,"uri":"/crucible/0/regions","method":"POST","req_id":"5f605059-6de5-45ae-9321-3981de186a88","remote_addr":"[fd00:1122:3344:101::c]:65371","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":67,"response_code":"200"}