[ Jan 14 22:31:22 Enabled. ]
[ Jan 14 22:31:22 Rereading configuration. ]
[ Jan 14 22:31:25 Rereading configuration. ]
[ Jan 14 22:31:28 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_ddc0f818-ddaf-43ed-a26d-e0c1b3bd5e59/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_ddc0f818-ddaf-43ed-a26d-e0c1b3bd5e59/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:28.906891036Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:28.910448828Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:28.910461698Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:28.930812311Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:29.000025329Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.24791787Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:47102"}
{"msg":"region e9c808c3-9c84-4bcc-b30f-d0cd950f3abd state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.272061119Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.277500384Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.277516154Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"uri":"/crucible/0/regions","method":"POST","req_id":"2a924017-59af-47c3-925f-aecfdee3cffb","remote_addr":"[fd00:1122:3344:101::a]:47102","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":11481,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_ddc0f818-ddaf-43ed-a26d-e0c1b3bd5e59/crucible/regions/e9c808c3-9c84-4bcc-b30f-d0cd950f3abd","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.287842088Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ddc0f818-ddaf-43ed-a26d-e0c1b3bd5e59/crucible/regions/e9c808c3-9c84-4bcc-b30f-d0cd950f3abd","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.287907238Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e9c808c3-9c84-4bcc-b30f-d0cd950f3abd\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e9c808c3-9c84-4bcc-b30f-d0cd950f3abd\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.357731643Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"region":"e9c808c3-9c84-4bcc-b30f-d0cd950f3abd","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e9c808c3-9c84-4bcc-b30f-d0cd950f3abd\" \"--data\" \"/data/regions/e9c808c3-9c84-4bcc-b30f-d0cd950f3abd\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.35998027Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"region":"e9c808c3-9c84-4bcc-b30f-d0cd950f3abd","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.497703567Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"region":"e9c808c3-9c84-4bcc-b30f-d0cd950f3abd","component":"worker"}
{"msg":"region e9c808c3-9c84-4bcc-b30f-d0cd950f3abd state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.497789878Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.497992341Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e9c808c3-9c84-4bcc-b30f-d0cd950f3abd","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.498399416Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.501795736Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"uri":"/crucible/0/regions","method":"POST","req_id":"650f3cec-2166-4f3c-b970-03ee3fd21b4b","remote_addr":"[fd00:1122:3344:101::a]:47102","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e9c808c3-9c84-4bcc-b30f-d0cd950f3abd","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.502269512Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.512871169Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e9c808c3-9c84-4bcc-b30f-d0cd950f3abd","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.515819573Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e9c808c3-9c84-4bcc-b30f-d0cd950f3abd","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.51637934Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.516444111Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.516484511Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.516528732Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.521525151Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.526267838Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:39.702249456Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:49893"}
{"msg":"region 68fe51a9-c1c3-4953-9647-01a621d2df35 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:39.702377767Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:39.703096216Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:39.703129836Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"uri":"/crucible/0/regions","method":"POST","req_id":"35ea73c9-00a0-42cf-b908-d62d17f4989f","remote_addr":"[fd00:1122:3344:101::a]:49893","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":755,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_ddc0f818-ddaf-43ed-a26d-e0c1b3bd5e59/crucible/regions/68fe51a9-c1c3-4953-9647-01a621d2df35","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:39.716470906Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ddc0f818-ddaf-43ed-a26d-e0c1b3bd5e59/crucible/regions/68fe51a9-c1c3-4953-9647-01a621d2df35","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:39.716500206Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"68fe51a9-c1c3-4953-9647-01a621d2df35\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/68fe51a9-c1c3-4953-9647-01a621d2df35\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:39.827682916Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"region":"68fe51a9-c1c3-4953-9647-01a621d2df35","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"68fe51a9-c1c3-4953-9647-01a621d2df35\" \"--data\" \"/data/regions/68fe51a9-c1c3-4953-9647-01a621d2df35\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:39.827757497Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"region":"68fe51a9-c1c3-4953-9647-01a621d2df35","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:39.93504216Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"region":"68fe51a9-c1c3-4953-9647-01a621d2df35","component":"worker"}
{"msg":"region 68fe51a9-c1c3-4953-9647-01a621d2df35 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:39.935103911Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:39.935253382Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-68fe51a9-c1c3-4953-9647-01a621d2df35","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:39.935995311Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-68fe51a9-c1c3-4953-9647-01a621d2df35","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:39.942321757Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:39.95846298Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-68fe51a9-c1c3-4953-9647-01a621d2df35","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:39.962187945Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/68fe51a9-c1c3-4953-9647-01a621d2df35","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:39.962467358Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:39.962551329Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:39.96261481Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:39.962671541Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:39.967517228Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:39.978606951Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.010932658Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"uri":"/crucible/0/regions","method":"POST","req_id":"ef24add3-153b-4abc-a9ad-bd035f473d53","remote_addr":"[fd00:1122:3344:101::a]:49893","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":955,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.7301792Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"uri":"/crucible/0/regions/e9c808c3-9c84-4bcc-b30f-d0cd950f3abd","method":"GET","req_id":"9f71e8b4-9d16-4404-a3b1-889416dd3a1a","remote_addr":"[fd00:1122:3344:101::a]:49893","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":2878,"response_code":"200"}
{"msg":"path is \"/data/regions/e9c808c3-9c84-4bcc-b30f-d0cd950f3abd\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.735935188Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"datafile"}
{"msg":"dataset is oxp_ddc0f818-ddaf-43ed-a26d-e0c1b3bd5e59/crucible/regions/e9c808c3-9c84-4bcc-b30f-d0cd950f3abd","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.745550593Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"datafile"}
{"msg":"snapshot is oxp_ddc0f818-ddaf-43ed-a26d-e0c1b3bd5e59/crucible/regions/e9c808c3-9c84-4bcc-b30f-d0cd950f3abd@ed7e43c3-4205-4e31-b98f-42e7a917cf19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.754250147Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ddc0f818-ddaf-43ed-a26d-e0c1b3bd5e59/crucible/regions/e9c808c3-9c84-4bcc-b30f-d0cd950f3abd\", \"ed7e43c3-4205-4e31-b98f-42e7a917cf19\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.754280278Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.765260969Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"uri":"/crucible/0/regions/e9c808c3-9c84-4bcc-b30f-d0cd950f3abd/snapshots/ed7e43c3-4205-4e31-b98f-42e7a917cf19","method":"GET","req_id":"a60eda22-470f-459b-8a11-a4a3b54014f3","remote_addr":"[fd00:1122:3344:101::a]:49893","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":29333,"response_code":"200"}
{"msg":"path is \"/data/regions/e9c808c3-9c84-4bcc-b30f-d0cd950f3abd\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.76866236Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"datafile"}
{"msg":"dataset is oxp_ddc0f818-ddaf-43ed-a26d-e0c1b3bd5e59/crucible/regions/e9c808c3-9c84-4bcc-b30f-d0cd950f3abd","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.778028522Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"datafile"}
{"msg":"snapshot is oxp_ddc0f818-ddaf-43ed-a26d-e0c1b3bd5e59/crucible/regions/e9c808c3-9c84-4bcc-b30f-d0cd950f3abd@ed7e43c3-4205-4e31-b98f-42e7a917cf19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.785868666Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ddc0f818-ddaf-43ed-a26d-e0c1b3bd5e59/crucible/regions/e9c808c3-9c84-4bcc-b30f-d0cd950f3abd\", \"ed7e43c3-4205-4e31-b98f-42e7a917cf19\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.785905386Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot e9c808c3-9c84-4bcc-b30f-d0cd950f3abd-ed7e43c3-4205-4e31-b98f-42e7a917cf19 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.794123584Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"datafile"}
{"msg":"applying SMF actions for region e9c808c3-9c84-4bcc-b30f-d0cd950f3abd running snapshot ed7e43c3-4205-4e31-b98f-42e7a917cf19 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.794326517Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.794363237Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"uri":"/crucible/0/regions/e9c808c3-9c84-4bcc-b30f-d0cd950f3abd/snapshots/ed7e43c3-4205-4e31-b98f-42e7a917cf19/run","method":"POST","req_id":"56b92763-94b3-43c1-940d-eb526d9409ea","remote_addr":"[fd00:1122:3344:101::a]:49893","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":25737,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-e9c808c3-9c84-4bcc-b30f-d0cd950f3abd-ed7e43c3-4205-4e31-b98f-42e7a917cf19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.796623154Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-e9c808c3-9c84-4bcc-b30f-d0cd950f3abd-ed7e43c3-4205-4e31-b98f-42e7a917cf19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.805366289Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-e9c808c3-9c84-4bcc-b30f-d0cd950f3abd-ed7e43c3-4205-4e31-b98f-42e7a917cf19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.816235779Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e9c808c3-9c84-4bcc-b30f-d0cd950f3abd/.zfs/snapshot/ed7e43c3-4205-4e31-b98f-42e7a917cf19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.816479232Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.816532162Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.816576333Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.816615653Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.816665874Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.819807421Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.824416867Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"region e9c808c3-9c84-4bcc-b30f-d0cd950f3abd running snapshot ed7e43c3-4205-4e31-b98f-42e7a917cf19 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.824431867Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"datafile"}
{"msg":"path is \"/data/regions/e9c808c3-9c84-4bcc-b30f-d0cd950f3abd\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.94009484Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"datafile"}
{"msg":"dataset is oxp_ddc0f818-ddaf-43ed-a26d-e0c1b3bd5e59/crucible/regions/e9c808c3-9c84-4bcc-b30f-d0cd950f3abd","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.948603372Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"datafile"}
{"msg":"snapshot is oxp_ddc0f818-ddaf-43ed-a26d-e0c1b3bd5e59/crucible/regions/e9c808c3-9c84-4bcc-b30f-d0cd950f3abd@ed7e43c3-4205-4e31-b98f-42e7a917cf19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.955879369Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ddc0f818-ddaf-43ed-a26d-e0c1b3bd5e59/crucible/regions/e9c808c3-9c84-4bcc-b30f-d0cd950f3abd\", \"ed7e43c3-4205-4e31-b98f-42e7a917cf19\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.955906539Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.963932735Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"uri":"/crucible/0/regions/e9c808c3-9c84-4bcc-b30f-d0cd950f3abd/snapshots/ed7e43c3-4205-4e31-b98f-42e7a917cf19/run","method":"POST","req_id":"878704aa-8cbb-4e21-bfd8-cd652b0a5423","remote_addr":"[fd00:1122:3344:101::a]:49893","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":23871,"response_code":"200"}
{"msg":"region dab285ba-53d1-42c3-9977-6fe943651653 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:45.14608433Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:45.146293122Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"uri":"/crucible/0/regions","method":"POST","req_id":"61fda4f3-451a-4faf-8694-e8ef17a1f8f0","remote_addr":"[fd00:1122:3344:101::a]:49893","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":280,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:45.146333263Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_ddc0f818-ddaf-43ed-a26d-e0c1b3bd5e59/crucible/regions/dab285ba-53d1-42c3-9977-6fe943651653","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:45.158035413Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ddc0f818-ddaf-43ed-a26d-e0c1b3bd5e59/crucible/regions/dab285ba-53d1-42c3-9977-6fe943651653","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:45.158064093Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"dab285ba-53d1-42c3-9977-6fe943651653\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/dab285ba-53d1-42c3-9977-6fe943651653\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:45.216132768Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"region":"dab285ba-53d1-42c3-9977-6fe943651653","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"dab285ba-53d1-42c3-9977-6fe943651653\" \"--data\" \"/data/regions/dab285ba-53d1-42c3-9977-6fe943651653\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:45.216185678Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"region":"dab285ba-53d1-42c3-9977-6fe943651653","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:45.291634871Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"region":"dab285ba-53d1-42c3-9977-6fe943651653","component":"worker"}
{"msg":"region dab285ba-53d1-42c3-9977-6fe943651653 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:45.291709652Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:45.291781062Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-dab285ba-53d1-42c3-9977-6fe943651653","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:45.293135318Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-dab285ba-53d1-42c3-9977-6fe943651653","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:45.296435778Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:45.304614706Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-dab285ba-53d1-42c3-9977-6fe943651653","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:45.307575781Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/dab285ba-53d1-42c3-9977-6fe943651653","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:45.307845454Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:45.307900215Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:45.307948066Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:45.307991276Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:45.311849842Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:45.327157155Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:45.510466568Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"uri":"/crucible/0/regions","method":"POST","req_id":"5ac7707e-1f14-4377-9a38-c963b1823701","remote_addr":"[fd00:1122:3344:101::a]:49893","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:42:10.004446793Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"uri":"/crucible/0/regions/dab285ba-53d1-42c3-9977-6fe943651653","method":"GET","req_id":"cdf03c47-7cb9-4815-a0a6-9b84f90418b7","remote_addr":"[fd00:1122:3344:101::a]:49893","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":117,"response_code":"200"}
{"msg":"path is \"/data/regions/dab285ba-53d1-42c3-9977-6fe943651653\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:42:10.007812934Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"datafile"}
{"msg":"dataset is oxp_ddc0f818-ddaf-43ed-a26d-e0c1b3bd5e59/crucible/regions/dab285ba-53d1-42c3-9977-6fe943651653","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:42:10.02092206Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"datafile"}
{"msg":"region dab285ba-53d1-42c3-9977-6fe943651653 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:42:10.031422276Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:42:10.031600548Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:42:10.031666469Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"uri":"/crucible/0/regions/dab285ba-53d1-42c3-9977-6fe943651653","method":"DELETE","req_id":"ebd379e9-5c3f-43e5-8483-825b1f3543a0","remote_addr":"[fd00:1122:3344:101::a]:49893","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":23885,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-dab285ba-53d1-42c3-9977-6fe943651653 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:42:10.03507621Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:42:10.035442284Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"uri":"/crucible/0/regions/dab285ba-53d1-42c3-9977-6fe943651653","method":"GET","req_id":"563892ca-109b-4d34-963e-b152d6ad08ec","remote_addr":"[fd00:1122:3344:101::a]:49893","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":32,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:42:10.043703993Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_ddc0f818-ddaf-43ed-a26d-e0c1b3bd5e59/crucible/regions/dab285ba-53d1-42c3-9977-6fe943651653\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:42:10.054985308Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"region":"dab285ba-53d1-42c3-9977-6fe943651653","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:42:10.094721713Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"uri":"/crucible/0/regions/dab285ba-53d1-42c3-9977-6fe943651653","method":"GET","req_id":"c69fee22-4fda-4afb-aaa1-19c054ed90a2","remote_addr":"[fd00:1122:3344:101::a]:49893","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":95,"response_code":"200"}
{"msg":"region dab285ba-53d1-42c3-9977-6fe943651653 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:42:10.109806653Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:42:10.243472382Z","hostname":"oxz_crucible_c1c1faca-128a-4fbc-9305-aff02770e047","pid":25998,"uri":"/crucible/0/regions/dab285ba-53d1-42c3-9977-6fe943651653","method":"GET","req_id":"55f9e66e-124d-4227-a546-0b78401044a6","remote_addr":"[fd00:1122:3344:101::a]:49893","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":43,"response_code":"200"}