[ Aug  1 01:16:18 Enabled. ]
[ Aug  1 01:16:18 Rereading configuration. ]
[ Aug  1 01:16:20 Rereading configuration. ]
[ Aug  1 01:16:22 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_30d0611c-3996-4f36-b74b-75dc9dd8a833/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_30d0611c-3996-4f36-b74b-75dc9dd8a833/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:22.509068559Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:22.522512599Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:22.522527689Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:22.55232414Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:22.669373396Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.206038775Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:34590"}
{"msg":"region 1b4089a6-77ad-496f-8951-18a0a665d777 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.22539812Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.231776524Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"uri":"/crucible/0/regions","method":"POST","req_id":"1dddad9e-5f51-4805-ac7e-16e06c55c9db","remote_addr":"[fd00:1122:3344:101::a]:34590","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":14797,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.232364954Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_30d0611c-3996-4f36-b74b-75dc9dd8a833/crucible/regions/1b4089a6-77ad-496f-8951-18a0a665d777","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.244887897Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_30d0611c-3996-4f36-b74b-75dc9dd8a833/crucible/regions/1b4089a6-77ad-496f-8951-18a0a665d777","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.244924188Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"1b4089a6-77ad-496f-8951-18a0a665d777\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/1b4089a6-77ad-496f-8951-18a0a665d777\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.294997941Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"region":"1b4089a6-77ad-496f-8951-18a0a665d777","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"1b4089a6-77ad-496f-8951-18a0a665d777\" \"--data\" \"/data/regions/1b4089a6-77ad-496f-8951-18a0a665d777\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.296976586Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"region":"1b4089a6-77ad-496f-8951-18a0a665d777","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.370423704Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"uri":"/crucible/0/regions","method":"POST","req_id":"7982b3e9-1113-4a4f-b009-eeaa37596500","remote_addr":"[fd00:1122:3344:101::a]:34590","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":86,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.436131835Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"region":"1b4089a6-77ad-496f-8951-18a0a665d777","component":"worker"}
{"msg":"region 1b4089a6-77ad-496f-8951-18a0a665d777 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.436168786Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.436304698Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-1b4089a6-77ad-496f-8951-18a0a665d777","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.436724176Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-1b4089a6-77ad-496f-8951-18a0a665d777","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.44257523Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.457201711Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1b4089a6-77ad-496f-8951-18a0a665d777","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.460355927Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1b4089a6-77ad-496f-8951-18a0a665d777","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.46108984Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.465077251Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.465133202Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.465180403Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.472099156Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.479489858Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:36.028001105Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"uri":"/crucible/0/regions","method":"POST","req_id":"a0760a2e-226c-4ba4-a1d7-0b0990133f4a","remote_addr":"[fd00:1122:3344:101::a]:34590","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":98,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.232558778Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:58402"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.235211566Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"uri":"/crucible/0/regions/1b4089a6-77ad-496f-8951-18a0a665d777","method":"GET","req_id":"5f1c4e35-c97a-442e-8b0a-2ea8c5b3425d","remote_addr":"[fd00:1122:3344:101::a]:58402","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":2611,"response_code":"200"}
{"msg":"path is \"/data/regions/1b4089a6-77ad-496f-8951-18a0a665d777\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.237513937Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"datafile"}
{"msg":"dataset is oxp_30d0611c-3996-4f36-b74b-75dc9dd8a833/crucible/regions/1b4089a6-77ad-496f-8951-18a0a665d777","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.24781718Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"datafile"}
{"msg":"snapshot is oxp_30d0611c-3996-4f36-b74b-75dc9dd8a833/crucible/regions/1b4089a6-77ad-496f-8951-18a0a665d777@88052840-5a63-42fd-bb55-9316b8dcf787","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.259337296Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_30d0611c-3996-4f36-b74b-75dc9dd8a833/crucible/regions/1b4089a6-77ad-496f-8951-18a0a665d777\", \"88052840-5a63-42fd-bb55-9316b8dcf787\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.259378676Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.27250149Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"uri":"/crucible/0/regions/1b4089a6-77ad-496f-8951-18a0a665d777/snapshots/88052840-5a63-42fd-bb55-9316b8dcf787","method":"GET","req_id":"97ba15a1-721e-4d5a-94ac-0c3e81cbbaa6","remote_addr":"[fd00:1122:3344:101::a]:58402","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":37076,"response_code":"200"}
{"msg":"path is \"/data/regions/1b4089a6-77ad-496f-8951-18a0a665d777\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.272744315Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"datafile"}
{"msg":"dataset is oxp_30d0611c-3996-4f36-b74b-75dc9dd8a833/crucible/regions/1b4089a6-77ad-496f-8951-18a0a665d777","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.283198191Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"datafile"}
{"msg":"snapshot is oxp_30d0611c-3996-4f36-b74b-75dc9dd8a833/crucible/regions/1b4089a6-77ad-496f-8951-18a0a665d777@88052840-5a63-42fd-bb55-9316b8dcf787","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.294040504Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_30d0611c-3996-4f36-b74b-75dc9dd8a833/crucible/regions/1b4089a6-77ad-496f-8951-18a0a665d777\", \"88052840-5a63-42fd-bb55-9316b8dcf787\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.294084305Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 1b4089a6-77ad-496f-8951-18a0a665d777-88052840-5a63-42fd-bb55-9316b8dcf787 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.305643721Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"datafile"}
{"msg":"applying SMF actions for region 1b4089a6-77ad-496f-8951-18a0a665d777 running snapshot 88052840-5a63-42fd-bb55-9316b8dcf787 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.306486306Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.306553707Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"uri":"/crucible/0/regions/1b4089a6-77ad-496f-8951-18a0a665d777/snapshots/88052840-5a63-42fd-bb55-9316b8dcf787/run","method":"POST","req_id":"45ee5525-d510-4a28-a283-64bd031857ab","remote_addr":"[fd00:1122:3344:101::a]:58402","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":33850,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/1b4089a6-77ad-496f-8951-18a0a665d777","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.317335729Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.317471612Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.317601074Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.317614344Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.317680095Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1b4089a6-77ad-496f-8951-18a0a665d777","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.317752647Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1b4089a6-77ad-496f-8951-18a0a665d777","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.317895519Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.317993741Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.318068433Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.318147464Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.326066055Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-1b4089a6-77ad-496f-8951-18a0a665d777-88052840-5a63-42fd-bb55-9316b8dcf787","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.326840089Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-1b4089a6-77ad-496f-8951-18a0a665d777-88052840-5a63-42fd-bb55-9316b8dcf787","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.33536463Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-1b4089a6-77ad-496f-8951-18a0a665d777-88052840-5a63-42fd-bb55-9316b8dcf787","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.351165632Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1b4089a6-77ad-496f-8951-18a0a665d777/.zfs/snapshot/88052840-5a63-42fd-bb55-9316b8dcf787","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.351523578Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.3516324Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.351698041Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.351761812Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.351847654Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.35778234Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.362780069Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"region 1b4089a6-77ad-496f-8951-18a0a665d777 running snapshot 88052840-5a63-42fd-bb55-9316b8dcf787 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.362817219Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.092322157Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:61950"}
{"msg":"region f97488f5-bec6-41f1-8b37-3e3560f2251f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.093006999Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.095748078Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.095772569Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"uri":"/crucible/0/regions","method":"POST","req_id":"cace84bf-b9d3-43a5-a4d6-18bd62c6334f","remote_addr":"[fd00:1122:3344:101::a]:61950","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":3390,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_30d0611c-3996-4f36-b74b-75dc9dd8a833/crucible/regions/f97488f5-bec6-41f1-8b37-3e3560f2251f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.104856811Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_30d0611c-3996-4f36-b74b-75dc9dd8a833/crucible/regions/f97488f5-bec6-41f1-8b37-3e3560f2251f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.104883991Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f97488f5-bec6-41f1-8b37-3e3560f2251f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f97488f5-bec6-41f1-8b37-3e3560f2251f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.154903063Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"region":"f97488f5-bec6-41f1-8b37-3e3560f2251f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f97488f5-bec6-41f1-8b37-3e3560f2251f\" \"--data\" \"/data/regions/f97488f5-bec6-41f1-8b37-3e3560f2251f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.156944279Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"region":"f97488f5-bec6-41f1-8b37-3e3560f2251f","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.231148592Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"region":"f97488f5-bec6-41f1-8b37-3e3560f2251f","component":"worker"}
{"msg":"region f97488f5-bec6-41f1-8b37-3e3560f2251f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.231186582Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.231308984Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/1b4089a6-77ad-496f-8951-18a0a665d777","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.231891525Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.231975416Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.232056728Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.232063408Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.232107619Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1b4089a6-77ad-496f-8951-18a0a665d777","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.23215835Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1b4089a6-77ad-496f-8951-18a0a665d777","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.232683469Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.23277265Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.232837872Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.232896883Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.236554328Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f97488f5-bec6-41f1-8b37-3e3560f2251f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.236968215Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f97488f5-bec6-41f1-8b37-3e3560f2251f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.240273904Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.247435342Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f97488f5-bec6-41f1-8b37-3e3560f2251f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.250761271Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f97488f5-bec6-41f1-8b37-3e3560f2251f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.251048586Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.251124888Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.251170329Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.251219129Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.252889689Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"uri":"/crucible/0/regions","method":"POST","req_id":"45136b98-e2eb-4942-a8c0-7c8c4d6d47d8","remote_addr":"[fd00:1122:3344:101::a]:61950","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.25461605Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.260867251Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.826013431Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"uri":"/crucible/0/regions/f97488f5-bec6-41f1-8b37-3e3560f2251f","method":"GET","req_id":"2bad0bea-03ad-4b2c-8707-b0189287c6f5","remote_addr":"[fd00:1122:3344:101::a]:61950","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":71,"response_code":"200"}
{"msg":"path is \"/data/regions/f97488f5-bec6-41f1-8b37-3e3560f2251f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.828317372Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"datafile"}
{"msg":"dataset is oxp_30d0611c-3996-4f36-b74b-75dc9dd8a833/crucible/regions/f97488f5-bec6-41f1-8b37-3e3560f2251f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.838547494Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"datafile"}
{"msg":"region f97488f5-bec6-41f1-8b37-3e3560f2251f state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.846992075Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.847184698Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.847244899Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"uri":"/crucible/0/regions/f97488f5-bec6-41f1-8b37-3e3560f2251f","method":"DELETE","req_id":"32703194-6fc8-423b-b7d1-c982cca1f373","remote_addr":"[fd00:1122:3344:101::a]:61950","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":18949,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-f97488f5-bec6-41f1-8b37-3e3560f2251f (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.84840512Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.850231013Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"uri":"/crucible/0/regions/f97488f5-bec6-41f1-8b37-3e3560f2251f","method":"GET","req_id":"cc359d0d-5c31-4d4b-b311-27f6f618b68e","remote_addr":"[fd00:1122:3344:101::a]:61950","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":631,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/1b4089a6-77ad-496f-8951-18a0a665d777","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.85345562Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.853546362Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.853640873Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.853652654Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.853691904Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1b4089a6-77ad-496f-8951-18a0a665d777","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.853756376Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1b4089a6-77ad-496f-8951-18a0a665d777","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.854136402Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.854207853Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.854272195Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.854328936Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.858087193Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.859430386Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_30d0611c-3996-4f36-b74b-75dc9dd8a833/crucible/regions/f97488f5-bec6-41f1-8b37-3e3560f2251f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.869520397Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"region":"f97488f5-bec6-41f1-8b37-3e3560f2251f","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.904231975Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"uri":"/crucible/0/regions/f97488f5-bec6-41f1-8b37-3e3560f2251f","method":"GET","req_id":"65e83f8c-7088-40bb-add4-b3596fa50468","remote_addr":"[fd00:1122:3344:101::a]:61950","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":67,"response_code":"200"}
{"msg":"region f97488f5-bec6-41f1-8b37-3e3560f2251f state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.921939761Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:11.055025403Z","hostname":"oxz_crucible_5610ec76-4a00-422e-8013-a7c36ef4f4f4","pid":24079,"uri":"/crucible/0/regions/f97488f5-bec6-41f1-8b37-3e3560f2251f","method":"GET","req_id":"34e4b429-7182-4a3d-8a16-0dcb300e6d2f","remote_addr":"[fd00:1122:3344:101::a]:61950","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":23,"response_code":"200"}