[ Sep 23 18:20:58 Enabled. ]
[ Sep 23 18:20:58 Rereading configuration. ]
[ Sep 23 18:21:01 Rereading configuration. ]
[ Sep 23 18:21:08 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_1f7ede35-40db-4a1a-8087-fc87b6705700/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_1f7ede35-40db-4a1a-8087-fc87b6705700/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:21:08.692950506Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:21:08.695636233Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:21:08.695649953Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:21:08.710036634Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:21:08.975583056Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:22:34.473042581Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63933"}
{"msg":"region 5974210c-41ae-4667-b206-8171012260ea state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:22:34.510654064Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:22:34.51528725Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"uri":"/crucible/0/regions","method":"POST","req_id":"20de2100-4934-4f45-bc23-12a7a0bcf0c2","remote_addr":"[fd00:1122:3344:101::a]:63933","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":23878,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:22:34.519974427Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_1f7ede35-40db-4a1a-8087-fc87b6705700/crucible/regions/5974210c-41ae-4667-b206-8171012260ea","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:22:34.5284648Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_1f7ede35-40db-4a1a-8087-fc87b6705700/crucible/regions/5974210c-41ae-4667-b206-8171012260ea","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:22:34.528499411Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"5974210c-41ae-4667-b206-8171012260ea\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/5974210c-41ae-4667-b206-8171012260ea\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:22:34.582322519Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"region":"5974210c-41ae-4667-b206-8171012260ea","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"5974210c-41ae-4667-b206-8171012260ea\" \"--data\" \"/data/regions/5974210c-41ae-4667-b206-8171012260ea\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:22:34.586878213Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"region":"5974210c-41ae-4667-b206-8171012260ea","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:22:34.879013351Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"uri":"/crucible/0/regions","method":"POST","req_id":"a5a6eb98-6d40-4a84-bcdb-d944471e3183","remote_addr":"[fd00:1122:3344:101::a]:63933","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":83,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:22:34.892863558Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"region":"5974210c-41ae-4667-b206-8171012260ea","component":"worker"}
{"msg":"region 5974210c-41ae-4667-b206-8171012260ea state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:22:34.892891799Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:22:34.893045862Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-5974210c-41ae-4667-b206-8171012260ea","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:22:34.895931615Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-5974210c-41ae-4667-b206-8171012260ea","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:22:34.90131576Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:22:34.917998838Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5974210c-41ae-4667-b206-8171012260ea","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:22:34.921574297Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5974210c-41ae-4667-b206-8171012260ea","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:22:34.922825079Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:22:34.925373872Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:22:34.925438344Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:22:34.925478625Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:22:34.933773573Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:22:34.941439875Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:22:35.555744882Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"uri":"/crucible/0/regions","method":"POST","req_id":"72b46877-b814-4fc8-b5d8-562541e10b1f","remote_addr":"[fd00:1122:3344:101::a]:63933","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.607313586Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:48854"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.607428159Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"uri":"/crucible/0/regions/5974210c-41ae-4667-b206-8171012260ea","method":"GET","req_id":"f56fa261-f996-4ed1-81e1-a7a01cfab7d2","remote_addr":"[fd00:1122:3344:101::a]:48854","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":53,"response_code":"200"}
{"msg":"path is \"/data/regions/5974210c-41ae-4667-b206-8171012260ea\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.610372102Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"datafile"}
{"msg":"dataset is oxp_1f7ede35-40db-4a1a-8087-fc87b6705700/crucible/regions/5974210c-41ae-4667-b206-8171012260ea","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.6194694Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"datafile"}
{"msg":"snapshot is oxp_1f7ede35-40db-4a1a-8087-fc87b6705700/crucible/regions/5974210c-41ae-4667-b206-8171012260ea@bf6b8e04-a5bf-448b-a96c-5d70aa90f8a7","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.629068801Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_1f7ede35-40db-4a1a-8087-fc87b6705700/crucible/regions/5974210c-41ae-4667-b206-8171012260ea\", \"bf6b8e04-a5bf-448b-a96c-5d70aa90f8a7\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.629105942Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.637648416Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"uri":"/crucible/0/regions/5974210c-41ae-4667-b206-8171012260ea/snapshots/bf6b8e04-a5bf-448b-a96c-5d70aa90f8a7","method":"GET","req_id":"fe7f95d5-f35f-483a-b41d-8f373a93b7e5","remote_addr":"[fd00:1122:3344:101::a]:48854","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":27288,"response_code":"200"}
{"msg":"path is \"/data/regions/5974210c-41ae-4667-b206-8171012260ea\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.640888527Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"datafile"}
{"msg":"dataset is oxp_1f7ede35-40db-4a1a-8087-fc87b6705700/crucible/regions/5974210c-41ae-4667-b206-8171012260ea","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.648359264Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"datafile"}
{"msg":"snapshot is oxp_1f7ede35-40db-4a1a-8087-fc87b6705700/crucible/regions/5974210c-41ae-4667-b206-8171012260ea@bf6b8e04-a5bf-448b-a96c-5d70aa90f8a7","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.656161789Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_1f7ede35-40db-4a1a-8087-fc87b6705700/crucible/regions/5974210c-41ae-4667-b206-8171012260ea\", \"bf6b8e04-a5bf-448b-a96c-5d70aa90f8a7\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.656201081Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 5974210c-41ae-4667-b206-8171012260ea-bf6b8e04-a5bf-448b-a96c-5d70aa90f8a7 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.665048052Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"datafile"}
{"msg":"applying SMF actions for region 5974210c-41ae-4667-b206-8171012260ea running snapshot bf6b8e04-a5bf-448b-a96c-5d70aa90f8a7 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.665656837Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.665680618Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"uri":"/crucible/0/regions/5974210c-41ae-4667-b206-8171012260ea/snapshots/bf6b8e04-a5bf-448b-a96c-5d70aa90f8a7/run","method":"POST","req_id":"428bc156-c5c5-4350-b0a2-60391e578173","remote_addr":"[fd00:1122:3344:101::a]:48854","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":25377,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/5974210c-41ae-4667-b206-8171012260ea","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.678562911Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.678689994Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.678822897Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.678832377Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.678895509Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5974210c-41ae-4667-b206-8171012260ea","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.678964561Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5974210c-41ae-4667-b206-8171012260ea","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.679103084Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.681532685Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.681606787Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.681679869Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.689421733Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-5974210c-41ae-4667-b206-8171012260ea-bf6b8e04-a5bf-448b-a96c-5d70aa90f8a7","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.689912165Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-5974210c-41ae-4667-b206-8171012260ea-bf6b8e04-a5bf-448b-a96c-5d70aa90f8a7","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.693441623Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-5974210c-41ae-4667-b206-8171012260ea-bf6b8e04-a5bf-448b-a96c-5d70aa90f8a7","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.706041599Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5974210c-41ae-4667-b206-8171012260ea/.zfs/snapshot/bf6b8e04-a5bf-448b-a96c-5d70aa90f8a7","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.706233024Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.706298356Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.706345497Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.706395438Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.70645846Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.711089545Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.715753272Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"region 5974210c-41ae-4667-b206-8171012260ea running snapshot bf6b8e04-a5bf-448b-a96c-5d70aa90f8a7 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.715767943Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"datafile"}
{"msg":"path is \"/data/regions/5974210c-41ae-4667-b206-8171012260ea\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.827869921Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"datafile"}
{"msg":"dataset is oxp_1f7ede35-40db-4a1a-8087-fc87b6705700/crucible/regions/5974210c-41ae-4667-b206-8171012260ea","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.836518238Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"datafile"}
{"msg":"snapshot is oxp_1f7ede35-40db-4a1a-8087-fc87b6705700/crucible/regions/5974210c-41ae-4667-b206-8171012260ea@bf6b8e04-a5bf-448b-a96c-5d70aa90f8a7","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.845506412Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_1f7ede35-40db-4a1a-8087-fc87b6705700/crucible/regions/5974210c-41ae-4667-b206-8171012260ea\", \"bf6b8e04-a5bf-448b-a96c-5d70aa90f8a7\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.845557904Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:51.853466202Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"uri":"/crucible/0/regions/5974210c-41ae-4667-b206-8171012260ea/snapshots/bf6b8e04-a5bf-448b-a96c-5d70aa90f8a7/run","method":"POST","req_id":"a9cc69c5-1dea-498e-9dd3-4c031f05fdd8","remote_addr":"[fd00:1122:3344:101::a]:48854","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":25610,"response_code":"200"}
{"msg":"region 327e9947-f7e1-4b97-92df-e5d731ca17a8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.64993785Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.650131695Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"uri":"/crucible/0/regions","method":"POST","req_id":"d6536e9c-a4f5-4595-8c3b-cf6f363be20d","remote_addr":"[fd00:1122:3344:101::a]:48854","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":257,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.650175396Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_1f7ede35-40db-4a1a-8087-fc87b6705700/crucible/regions/327e9947-f7e1-4b97-92df-e5d731ca17a8","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.660102325Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_1f7ede35-40db-4a1a-8087-fc87b6705700/crucible/regions/327e9947-f7e1-4b97-92df-e5d731ca17a8","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.660125945Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"327e9947-f7e1-4b97-92df-e5d731ca17a8\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/327e9947-f7e1-4b97-92df-e5d731ca17a8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.734404376Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"region":"327e9947-f7e1-4b97-92df-e5d731ca17a8","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"327e9947-f7e1-4b97-92df-e5d731ca17a8\" \"--data\" \"/data/regions/327e9947-f7e1-4b97-92df-e5d731ca17a8\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.734465207Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"region":"327e9947-f7e1-4b97-92df-e5d731ca17a8","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.800699607Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"uri":"/crucible/0/regions","method":"POST","req_id":"40ead772-e133-49b8-a14a-80a8a4262486","remote_addr":"[fd00:1122:3344:101::a]:48854","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":85,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.811661831Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"region":"327e9947-f7e1-4b97-92df-e5d731ca17a8","component":"worker"}
{"msg":"region 327e9947-f7e1-4b97-92df-e5d731ca17a8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.811683362Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.811836706Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-327e9947-f7e1-4b97-92df-e5d731ca17a8","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.812320268Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-327e9947-f7e1-4b97-92df-e5d731ca17a8","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.817043366Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.825447487Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-327e9947-f7e1-4b97-92df-e5d731ca17a8","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.829445767Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/327e9947-f7e1-4b97-92df-e5d731ca17a8","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.829755725Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.829804686Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.829847247Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.829889498Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.834055152Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/5974210c-41ae-4667-b206-8171012260ea","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.840532224Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.840638537Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.84073768Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.84074789Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.840787711Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5974210c-41ae-4667-b206-8171012260ea","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.840835072Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5974210c-41ae-4667-b206-8171012260ea","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.844778311Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.844833362Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.844882544Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.844933025Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.849371576Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:55.851171131Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:27:56.282967706Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"uri":"/crucible/0/regions","method":"POST","req_id":"b63951c1-b141-4e37-b604-79935e7e115f","remote_addr":"[fd00:1122:3344:101::a]:48854","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":73,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:28:20.245401375Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"uri":"/crucible/0/regions/327e9947-f7e1-4b97-92df-e5d731ca17a8","method":"GET","req_id":"04a7d061-6e7e-4e7b-9046-d3c3cdd2cd3c","remote_addr":"[fd00:1122:3344:101::a]:48854","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":388,"response_code":"200"}
{"msg":"path is \"/data/regions/327e9947-f7e1-4b97-92df-e5d731ca17a8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:28:20.248244866Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"datafile"}
{"msg":"dataset is oxp_1f7ede35-40db-4a1a-8087-fc87b6705700/crucible/regions/327e9947-f7e1-4b97-92df-e5d731ca17a8","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:28:20.260239717Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"datafile"}
{"msg":"region 327e9947-f7e1-4b97-92df-e5d731ca17a8 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:28:20.270160865Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:28:20.2703652Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:28:20.270429902Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"uri":"/crucible/0/regions/327e9947-f7e1-4b97-92df-e5d731ca17a8","method":"DELETE","req_id":"df614139-8697-4a26-9ac6-39e05168f7d1","remote_addr":"[fd00:1122:3344:101::a]:48854","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":22209,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-327e9947-f7e1-4b97-92df-e5d731ca17a8 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:28:20.27155869Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:28:20.273656733Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"uri":"/crucible/0/regions/327e9947-f7e1-4b97-92df-e5d731ca17a8","method":"GET","req_id":"4c89481e-a837-4c69-928a-4baf20261149","remote_addr":"[fd00:1122:3344:101::a]:48854","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":30,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/5974210c-41ae-4667-b206-8171012260ea","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:28:20.277005057Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:28:20.277089219Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:28:20.277168961Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:28:20.277204582Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:28:20.277212252Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5974210c-41ae-4667-b206-8171012260ea","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:28:20.277268823Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5974210c-41ae-4667-b206-8171012260ea","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:28:20.277596162Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:28:20.277658483Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:28:20.277702094Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:28:20.277749705Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:28:20.282353511Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:28:20.284015622Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_1f7ede35-40db-4a1a-8087-fc87b6705700/crucible/regions/327e9947-f7e1-4b97-92df-e5d731ca17a8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:28:20.297038238Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"region":"327e9947-f7e1-4b97-92df-e5d731ca17a8","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:28:20.340489997Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"uri":"/crucible/0/regions/327e9947-f7e1-4b97-92df-e5d731ca17a8","method":"GET","req_id":"1371b480-20c5-471c-9faf-c3b508816b4a","remote_addr":"[fd00:1122:3344:101::a]:48854","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":89,"response_code":"200"}
{"msg":"region 327e9947-f7e1-4b97-92df-e5d731ca17a8 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:28:20.365537214Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T18:28:20.482464023Z","hostname":"oxz_crucible_44aeefad-3a65-438e-8790-1c7a579bbb49","pid":22129,"uri":"/crucible/0/regions/327e9947-f7e1-4b97-92df-e5d731ca17a8","method":"GET","req_id":"3080b523-ba6f-4c26-8ff5-f85ea7d8f6bd","remote_addr":"[fd00:1122:3344:101::a]:48854","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":43,"response_code":"200"}