[ Oct 22 17:00:01 Enabled. ]
[ Oct 22 17:00:01 Rereading configuration. ]
[ Oct 22 17:00:03 Rereading configuration. ]
[ Oct 22 17:00:05 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_7692978e-e7ef-41d7-a73a-73e0c38244a5/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_7692978e-e7ef-41d7-a73a-73e0c38244a5/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:00:05.186365021Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:00:05.188467649Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:00:05.188479459Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:00:05.209824538Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:00:05.369346794Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:08.785691231Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:39669"}
{"msg":"region 8801830e-925e-4af3-9784-e7a429fe8b38 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:08.820722464Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:08.823506278Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"uri":"/crucible/0/regions","method":"POST","req_id":"07aee705-684b-488e-a38a-472e7d5a3723","remote_addr":"[fd00:1122:3344:101::b]:39669","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":11796,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:08.828299948Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_7692978e-e7ef-41d7-a73a-73e0c38244a5/crucible/regions/8801830e-925e-4af3-9784-e7a429fe8b38","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:08.836794022Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_7692978e-e7ef-41d7-a73a-73e0c38244a5/crucible/regions/8801830e-925e-4af3-9784-e7a429fe8b38","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:08.836827043Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"8801830e-925e-4af3-9784-e7a429fe8b38\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/8801830e-925e-4af3-9784-e7a429fe8b38\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:08.888700782Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"region":"8801830e-925e-4af3-9784-e7a429fe8b38","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"8801830e-925e-4af3-9784-e7a429fe8b38\" \"--data\" \"/data/regions/8801830e-925e-4af3-9784-e7a429fe8b38\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:08.893136854Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"region":"8801830e-925e-4af3-9784-e7a429fe8b38","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:09.133185975Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"uri":"/crucible/0/regions","method":"POST","req_id":"3c9ef411-5fad-480c-b7c0-7b526eedd44a","remote_addr":"[fd00:1122:3344:101::b]:39669","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:09.143906481Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"region":"8801830e-925e-4af3-9784-e7a429fe8b38","component":"worker"}
{"msg":"region 8801830e-925e-4af3-9784-e7a429fe8b38 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:09.143932971Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:09.144067054Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-8801830e-925e-4af3-9784-e7a429fe8b38","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:09.144513215Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-8801830e-925e-4af3-9784-e7a429fe8b38","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:09.148423574Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:09.165027505Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-8801830e-925e-4af3-9784-e7a429fe8b38","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:09.168069415Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8801830e-925e-4af3-9784-e7a429fe8b38","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:09.176249562Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:09.176324914Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:09.176367114Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:09.176414246Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:09.183713213Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:09.189128317Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:09.546601649Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"uri":"/crucible/0/regions","method":"POST","req_id":"0cf125e6-dace-4c51-88e6-bded62bbf1a1","remote_addr":"[fd00:1122:3344:101::b]:39669","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":62,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:10.23000501Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"uri":"/crucible/0/regions/8801830e-925e-4af3-9784-e7a429fe8b38","method":"GET","req_id":"113e3be7-68df-431f-b7ad-9a5ddb4e77cf","remote_addr":"[fd00:1122:3344:101::b]:39669","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"path is \"/data/regions/8801830e-925e-4af3-9784-e7a429fe8b38\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:10.234255138Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"datafile"}
{"msg":"dataset is oxp_7692978e-e7ef-41d7-a73a-73e0c38244a5/crucible/regions/8801830e-925e-4af3-9784-e7a429fe8b38","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:10.244225376Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"datafile"}
{"msg":"region 8801830e-925e-4af3-9784-e7a429fe8b38 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:10.253416697Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:10.25356483Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:10.253644982Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"uri":"/crucible/0/regions/8801830e-925e-4af3-9784-e7a429fe8b38","method":"DELETE","req_id":"ae70b53a-516c-4530-9228-985c9c893f1f","remote_addr":"[fd00:1122:3344:101::b]:39669","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":19413,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-8801830e-925e-4af3-9784-e7a429fe8b38 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:10.254897031Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:10.256429256Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"uri":"/crucible/0/regions/8801830e-925e-4af3-9784-e7a429fe8b38","method":"GET","req_id":"7d334fab-57b4-48f1-b280-2af4fabfccd0","remote_addr":"[fd00:1122:3344:101::b]:39669","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":25,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:10.258705248Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_7692978e-e7ef-41d7-a73a-73e0c38244a5/crucible/regions/8801830e-925e-4af3-9784-e7a429fe8b38\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:10.268427331Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"region":"8801830e-925e-4af3-9784-e7a429fe8b38","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:10.312482621Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"uri":"/crucible/0/regions/8801830e-925e-4af3-9784-e7a429fe8b38","method":"GET","req_id":"7b4a5e72-5571-4914-bb74-e3ae3b809748","remote_addr":"[fd00:1122:3344:101::b]:39669","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":92,"response_code":"200"}
{"msg":"region 8801830e-925e-4af3-9784-e7a429fe8b38 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:10.325838316Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:10.418215074Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"uri":"/crucible/0/regions/8801830e-925e-4af3-9784-e7a429fe8b38","method":"GET","req_id":"60271130-b07f-453c-8822-65604abf98a7","remote_addr":"[fd00:1122:3344:101::b]:39669","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":32,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:19.983769689Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:43767"}
{"msg":"region 18ce6265-16e3-499b-aff6-ac5934d4eab5 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:19.983947203Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:19.984113327Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"uri":"/crucible/0/regions","method":"POST","req_id":"f5a7ee6b-1e7a-4b59-86cb-8a29db3bdf75","remote_addr":"[fd00:1122:3344:101::a]:43767","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":196,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:19.984144147Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_7692978e-e7ef-41d7-a73a-73e0c38244a5/crucible/regions/18ce6265-16e3-499b-aff6-ac5934d4eab5","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:19.993625665Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_7692978e-e7ef-41d7-a73a-73e0c38244a5/crucible/regions/18ce6265-16e3-499b-aff6-ac5934d4eab5","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:19.993658075Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"18ce6265-16e3-499b-aff6-ac5934d4eab5\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/18ce6265-16e3-499b-aff6-ac5934d4eab5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:20.047696624Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"region":"18ce6265-16e3-499b-aff6-ac5934d4eab5","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"18ce6265-16e3-499b-aff6-ac5934d4eab5\" \"--data\" \"/data/regions/18ce6265-16e3-499b-aff6-ac5934d4eab5\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:20.049471674Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"region":"18ce6265-16e3-499b-aff6-ac5934d4eab5","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:20.118564928Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"region":"18ce6265-16e3-499b-aff6-ac5934d4eab5","component":"worker"}
{"msg":"region 18ce6265-16e3-499b-aff6-ac5934d4eab5 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:20.118602139Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:20.118704091Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-18ce6265-16e3-499b-aff6-ac5934d4eab5","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:20.120062822Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-18ce6265-16e3-499b-aff6-ac5934d4eab5","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:20.123590883Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:20.131218038Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-18ce6265-16e3-499b-aff6-ac5934d4eab5","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:20.13480758Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/18ce6265-16e3-499b-aff6-ac5934d4eab5","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:20.135031385Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:20.135088576Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:20.135133518Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:20.135174889Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:20.138690279Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:20.144662026Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:01:20.200149427Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"uri":"/crucible/0/regions","method":"POST","req_id":"c707f246-fd8f-4210-b5fa-096bd38c70bf","remote_addr":"[fd00:1122:3344:101::a]:43767","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":83,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:43.800523392Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:52426"}
{"msg":"region 982b88da-5f31-4e53-a520-1bd1c973ec50 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:43.800692286Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:43.806157742Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"uri":"/crucible/0/regions","method":"POST","req_id":"906e79f9-af83-4e35-84b6-5fd45467fec6","remote_addr":"[fd00:1122:3344:101::a]:52426","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":5583,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:43.806183242Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_7692978e-e7ef-41d7-a73a-73e0c38244a5/crucible/regions/982b88da-5f31-4e53-a520-1bd1c973ec50","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:43.815910405Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_7692978e-e7ef-41d7-a73a-73e0c38244a5/crucible/regions/982b88da-5f31-4e53-a520-1bd1c973ec50","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:43.815945946Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"982b88da-5f31-4e53-a520-1bd1c973ec50\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/982b88da-5f31-4e53-a520-1bd1c973ec50\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:44.077083931Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"region":"982b88da-5f31-4e53-a520-1bd1c973ec50","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"982b88da-5f31-4e53-a520-1bd1c973ec50\" \"--data\" \"/data/regions/982b88da-5f31-4e53-a520-1bd1c973ec50\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:44.078881262Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"region":"982b88da-5f31-4e53-a520-1bd1c973ec50","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:44.130171817Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"uri":"/crucible/0/regions","method":"POST","req_id":"788ddedd-8845-4268-88a2-8dc8d5725b7a","remote_addr":"[fd00:1122:3344:101::a]:52426","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":125,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:44.547663004Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"region":"982b88da-5f31-4e53-a520-1bd1c973ec50","component":"worker"}
{"msg":"region 982b88da-5f31-4e53-a520-1bd1c973ec50 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:44.547727486Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:44.547827778Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-982b88da-5f31-4e53-a520-1bd1c973ec50","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:44.571688585Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-982b88da-5f31-4e53-a520-1bd1c973ec50","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:44.602608714Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:44.681929861Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-982b88da-5f31-4e53-a520-1bd1c973ec50","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:44.767630336Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/982b88da-5f31-4e53-a520-1bd1c973ec50","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:44.768014304Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:44.768071456Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:44.768107636Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:44.768157698Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:44.803367335Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:44.835205804Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"uri":"/crucible/0/regions","method":"POST","req_id":"f757216b-caac-452e-bfc7-2c131d27321e","remote_addr":"[fd00:1122:3344:101::a]:52426","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:44.898082715Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.341503896Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"uri":"/crucible/0/regions/18ce6265-16e3-499b-aff6-ac5934d4eab5","method":"GET","req_id":"b412f6f5-3647-40ca-a88a-88fee2966fcb","remote_addr":"[fd00:1122:3344:101::a]:52426","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":59,"response_code":"200"}
{"msg":"path is \"/data/regions/18ce6265-16e3-499b-aff6-ac5934d4eab5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.348616739Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"datafile"}
{"msg":"dataset is oxp_7692978e-e7ef-41d7-a73a-73e0c38244a5/crucible/regions/18ce6265-16e3-499b-aff6-ac5934d4eab5","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.357255257Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"datafile"}
{"msg":"snapshot is oxp_7692978e-e7ef-41d7-a73a-73e0c38244a5/crucible/regions/18ce6265-16e3-499b-aff6-ac5934d4eab5@7a7b6ddb-1867-48c9-83ba-43b17a4f20f0","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.366042408Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_7692978e-e7ef-41d7-a73a-73e0c38244a5/crucible/regions/18ce6265-16e3-499b-aff6-ac5934d4eab5\", \"7a7b6ddb-1867-48c9-83ba-43b17a4f20f0\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.366081819Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.379203809Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"uri":"/crucible/0/regions/18ce6265-16e3-499b-aff6-ac5934d4eab5/snapshots/7a7b6ddb-1867-48c9-83ba-43b17a4f20f0","method":"GET","req_id":"18e47ea6-0347-466f-908f-fb8f07cda574","remote_addr":"[fd00:1122:3344:101::a]:52426","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":30601,"response_code":"200"}
{"msg":"path is \"/data/regions/18ce6265-16e3-499b-aff6-ac5934d4eab5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.382736461Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"datafile"}
{"msg":"dataset is oxp_7692978e-e7ef-41d7-a73a-73e0c38244a5/crucible/regions/18ce6265-16e3-499b-aff6-ac5934d4eab5","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.391469441Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"datafile"}
{"msg":"snapshot is oxp_7692978e-e7ef-41d7-a73a-73e0c38244a5/crucible/regions/18ce6265-16e3-499b-aff6-ac5934d4eab5@7a7b6ddb-1867-48c9-83ba-43b17a4f20f0","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.400868306Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_7692978e-e7ef-41d7-a73a-73e0c38244a5/crucible/regions/18ce6265-16e3-499b-aff6-ac5934d4eab5\", \"7a7b6ddb-1867-48c9-83ba-43b17a4f20f0\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.400910167Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 18ce6265-16e3-499b-aff6-ac5934d4eab5-7a7b6ddb-1867-48c9-83ba-43b17a4f20f0 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.409597426Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"datafile"}
{"msg":"applying SMF actions for region 18ce6265-16e3-499b-aff6-ac5934d4eab5 running snapshot 7a7b6ddb-1867-48c9-83ba-43b17a4f20f0 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.409798371Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.409818141Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"uri":"/crucible/0/regions/18ce6265-16e3-499b-aff6-ac5934d4eab5/snapshots/7a7b6ddb-1867-48c9-83ba-43b17a4f20f0/run","method":"POST","req_id":"ffa4d64f-f06d-4c55-adb3-7421415c5185","remote_addr":"[fd00:1122:3344:101::a]:52426","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":27116,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-18ce6265-16e3-499b-aff6-ac5934d4eab5-7a7b6ddb-1867-48c9-83ba-43b17a4f20f0","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.412574654Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-18ce6265-16e3-499b-aff6-ac5934d4eab5-7a7b6ddb-1867-48c9-83ba-43b17a4f20f0","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.43114653Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-18ce6265-16e3-499b-aff6-ac5934d4eab5-7a7b6ddb-1867-48c9-83ba-43b17a4f20f0","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.483653143Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/18ce6265-16e3-499b-aff6-ac5934d4eab5/.zfs/snapshot/7a7b6ddb-1867-48c9-83ba-43b17a4f20f0","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.48396311Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.484015102Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.484055543Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.484095813Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.484148414Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.505034843Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.530935687Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"worker"}
{"msg":"region 18ce6265-16e3-499b-aff6-ac5934d4eab5 running snapshot 7a7b6ddb-1867-48c9-83ba-43b17a4f20f0 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.530953697Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"datafile"}
{"msg":"path is \"/data/regions/18ce6265-16e3-499b-aff6-ac5934d4eab5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.786372431Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"datafile"}
{"msg":"dataset is oxp_7692978e-e7ef-41d7-a73a-73e0c38244a5/crucible/regions/18ce6265-16e3-499b-aff6-ac5934d4eab5","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.795024499Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"datafile"}
{"msg":"snapshot is oxp_7692978e-e7ef-41d7-a73a-73e0c38244a5/crucible/regions/18ce6265-16e3-499b-aff6-ac5934d4eab5@7a7b6ddb-1867-48c9-83ba-43b17a4f20f0","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.802807657Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_7692978e-e7ef-41d7-a73a-73e0c38244a5/crucible/regions/18ce6265-16e3-499b-aff6-ac5934d4eab5\", \"7a7b6ddb-1867-48c9-83ba-43b17a4f20f0\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.802845788Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-22T17:04:49.810280879Z","hostname":"oxz_crucible_6f4facaf-1abc-45e6-ab54-7b4a5c7e2f61","pid":26385,"uri":"/crucible/0/regions/18ce6265-16e3-499b-aff6-ac5934d4eab5/snapshots/7a7b6ddb-1867-48c9-83ba-43b17a4f20f0/run","method":"POST","req_id":"8d28c1e2-68e4-46ad-b29d-1493ec8a42df","remote_addr":"[fd00:1122:3344:101::a]:52426","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":24485,"response_code":"200"}