[ Jun 27 21:49:54 Enabled. ]
[ Jun 27 21:49:54 Rereading configuration. ]
[ Jun 27 21:49:57 Rereading configuration. ]
[ Jun 27 21:50:03 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_f8b5491e-78ae-4725-a4e0-5d2f23407448/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_f8b5491e-78ae-4725-a4e0-5d2f23407448/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:50:03.873061363Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:50:03.876309569Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:50:03.876322066Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:50:03.890089009Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:50:04.053817714Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:07.707071305Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:49139"}
{"msg":"region 2ab3593d-c760-43b8-a461-f585a7b2dfed state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:07.729449682Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:07.737661915Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"uri":"/crucible/0/regions","method":"POST","req_id":"c55f5d5a-cdc7-4189-bf20-0bf40ce356c3","remote_addr":"[fd00:1122:3344:101::a]:49139","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":18114,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:07.739559457Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_f8b5491e-78ae-4725-a4e0-5d2f23407448/crucible/regions/2ab3593d-c760-43b8-a461-f585a7b2dfed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:07.750044701Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_f8b5491e-78ae-4725-a4e0-5d2f23407448/crucible/regions/2ab3593d-c760-43b8-a461-f585a7b2dfed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:07.750076852Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"2ab3593d-c760-43b8-a461-f585a7b2dfed\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/2ab3593d-c760-43b8-a461-f585a7b2dfed\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:07.803888773Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"region":"2ab3593d-c760-43b8-a461-f585a7b2dfed","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"2ab3593d-c760-43b8-a461-f585a7b2dfed\" \"--data\" \"/data/regions/2ab3593d-c760-43b8-a461-f585a7b2dfed\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:07.806209626Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"region":"2ab3593d-c760-43b8-a461-f585a7b2dfed","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:07.999262425Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"uri":"/crucible/0/regions","method":"POST","req_id":"89423b8b-699d-4797-b7f8-fc836ff70a4a","remote_addr":"[fd00:1122:3344:101::a]:49139","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":87,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.042722715Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"region":"2ab3593d-c760-43b8-a461-f585a7b2dfed","component":"worker"}
{"msg":"region 2ab3593d-c760-43b8-a461-f585a7b2dfed state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.042764866Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.042858649Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-2ab3593d-c760-43b8-a461-f585a7b2dfed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.043257599Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-2ab3593d-c760-43b8-a461-f585a7b2dfed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.050842005Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.069850551Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2ab3593d-c760-43b8-a461-f585a7b2dfed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.073629554Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2ab3593d-c760-43b8-a461-f585a7b2dfed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.074346553Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.07644853Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.076509382Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.076550043Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.08638092Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.093243816Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.499817231Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"uri":"/crucible/0/regions","method":"POST","req_id":"514ff51a-48d1-4a89-bc91-af1bc3e84307","remote_addr":"[fd00:1122:3344:101::a]:49139","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.787021667Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"uri":"/crucible/0/regions/2ab3593d-c760-43b8-a461-f585a7b2dfed","method":"GET","req_id":"3a211638-6008-459f-a198-73edd9df1b5e","remote_addr":"[fd00:1122:3344:101::a]:49139","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"path is \"/data/regions/2ab3593d-c760-43b8-a461-f585a7b2dfed\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.789602947Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"datafile"}
{"msg":"dataset is oxp_f8b5491e-78ae-4725-a4e0-5d2f23407448/crucible/regions/2ab3593d-c760-43b8-a461-f585a7b2dfed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.798903809Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"datafile"}
{"msg":"region 2ab3593d-c760-43b8-a461-f585a7b2dfed state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.808547721Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.808701885Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.808765437Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"uri":"/crucible/0/regions/2ab3593d-c760-43b8-a461-f585a7b2dfed","method":"DELETE","req_id":"d4a9d0f8-ac93-4e84-be18-99f3070e777a","remote_addr":"[fd00:1122:3344:101::a]:49139","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":19221,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-2ab3593d-c760-43b8-a461-f585a7b2dfed (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.809816695Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.810631808Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"uri":"/crucible/0/regions/2ab3593d-c760-43b8-a461-f585a7b2dfed","method":"GET","req_id":"b28b0958-3917-485f-9c5e-9560229bab65","remote_addr":"[fd00:1122:3344:101::a]:49139","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":23,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.813830675Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_f8b5491e-78ae-4725-a4e0-5d2f23407448/crucible/regions/2ab3593d-c760-43b8-a461-f585a7b2dfed\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.823342753Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"region":"2ab3593d-c760-43b8-a461-f585a7b2dfed","component":"worker"}
{"msg":"region 2ab3593d-c760-43b8-a461-f585a7b2dfed state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.872784615Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:08.910109818Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"uri":"/crucible/0/regions/2ab3593d-c760-43b8-a461-f585a7b2dfed","method":"GET","req_id":"39a11ada-a5f1-4da3-aa5a-c0d276be7be0","remote_addr":"[fd00:1122:3344:101::a]:49139","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":48,"response_code":"200"}
{"msg":"region a165be3c-2dc9-4e51-a247-96660f2a91ef state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:10.394393994Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:10.394563639Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:10.39459948Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"uri":"/crucible/0/regions","method":"POST","req_id":"88518fee-5051-4bc7-ab73-1fd9d975d329","remote_addr":"[fd00:1122:3344:101::a]:49139","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":250,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_f8b5491e-78ae-4725-a4e0-5d2f23407448/crucible/regions/a165be3c-2dc9-4e51-a247-96660f2a91ef","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:10.402696579Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_f8b5491e-78ae-4725-a4e0-5d2f23407448/crucible/regions/a165be3c-2dc9-4e51-a247-96660f2a91ef","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:10.40272691Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"a165be3c-2dc9-4e51-a247-96660f2a91ef\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a165be3c-2dc9-4e51-a247-96660f2a91ef\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:10.453906809Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"region":"a165be3c-2dc9-4e51-a247-96660f2a91ef","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a165be3c-2dc9-4e51-a247-96660f2a91ef\" \"--data\" \"/data/regions/a165be3c-2dc9-4e51-a247-96660f2a91ef\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:10.456000186Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"region":"a165be3c-2dc9-4e51-a247-96660f2a91ef","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:10.529693326Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"region":"a165be3c-2dc9-4e51-a247-96660f2a91ef","component":"worker"}
{"msg":"region a165be3c-2dc9-4e51-a247-96660f2a91ef state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:10.529733237Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:10.52982217Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-a165be3c-2dc9-4e51-a247-96660f2a91ef","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:10.531192857Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a165be3c-2dc9-4e51-a247-96660f2a91ef","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:10.534492717Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:10.542508294Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a165be3c-2dc9-4e51-a247-96660f2a91ef","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:10.545704411Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a165be3c-2dc9-4e51-a247-96660f2a91ef","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:10.546002019Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:10.546062341Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:10.546106572Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:10.546151223Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:10.550071749Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:10.555127527Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:51:10.747507708Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"uri":"/crucible/0/regions","method":"POST","req_id":"0f8d470e-286d-45db-b8a4-9fd3cb4bc78e","remote_addr":"[fd00:1122:3344:101::a]:49139","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":74,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:37.696877211Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:33626"}
{"msg":"region f311d1d5-5bbc-40aa-9816-920626e6637e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:37.697140008Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:37.698062603Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:37.698081503Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"uri":"/crucible/0/regions","method":"POST","req_id":"cd960255-5adb-49d9-82b3-befaf2a18157","remote_addr":"[fd00:1122:3344:101::a]:33626","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":949,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_f8b5491e-78ae-4725-a4e0-5d2f23407448/crucible/regions/f311d1d5-5bbc-40aa-9816-920626e6637e","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:37.708371523Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_f8b5491e-78ae-4725-a4e0-5d2f23407448/crucible/regions/f311d1d5-5bbc-40aa-9816-920626e6637e","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:37.708410104Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f311d1d5-5bbc-40aa-9816-920626e6637e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f311d1d5-5bbc-40aa-9816-920626e6637e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:37.764131016Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"region":"f311d1d5-5bbc-40aa-9816-920626e6637e","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f311d1d5-5bbc-40aa-9816-920626e6637e\" \"--data\" \"/data/regions/f311d1d5-5bbc-40aa-9816-920626e6637e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:37.764189638Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"region":"f311d1d5-5bbc-40aa-9816-920626e6637e","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:37.837535219Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"region":"f311d1d5-5bbc-40aa-9816-920626e6637e","component":"worker"}
{"msg":"region f311d1d5-5bbc-40aa-9816-920626e6637e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:37.83759818Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:37.837791376Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f311d1d5-5bbc-40aa-9816-920626e6637e","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:37.846385329Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f311d1d5-5bbc-40aa-9816-920626e6637e","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:37.853878502Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:37.86448333Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f311d1d5-5bbc-40aa-9816-920626e6637e","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:37.867570074Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f311d1d5-5bbc-40aa-9816-920626e6637e","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:37.867954924Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:37.868021316Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:37.868136309Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:37.86815158Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:37.87185519Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:37.878232963Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:37.946195838Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"uri":"/crucible/0/regions","method":"POST","req_id":"1f02b48c-2aff-43b5-915d-d31ce3eccbb1","remote_addr":"[fd00:1122:3344:101::a]:33626","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.414515189Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:40440"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.414632662Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"uri":"/crucible/0/regions/a165be3c-2dc9-4e51-a247-96660f2a91ef","method":"GET","req_id":"0c817aa2-cb4c-4e06-b4b4-aff096ce2050","remote_addr":"[fd00:1122:3344:101::a]:40440","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":52,"response_code":"200"}
{"msg":"path is \"/data/regions/a165be3c-2dc9-4e51-a247-96660f2a91ef\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.414815637Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"datafile"}
{"msg":"dataset is oxp_f8b5491e-78ae-4725-a4e0-5d2f23407448/crucible/regions/a165be3c-2dc9-4e51-a247-96660f2a91ef","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.425839016Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"datafile"}
{"msg":"snapshot is oxp_f8b5491e-78ae-4725-a4e0-5d2f23407448/crucible/regions/a165be3c-2dc9-4e51-a247-96660f2a91ef@bd357839-a321-4a4a-b806-b5f3cf68741c","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.434305426Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_f8b5491e-78ae-4725-a4e0-5d2f23407448/crucible/regions/a165be3c-2dc9-4e51-a247-96660f2a91ef\", \"bd357839-a321-4a4a-b806-b5f3cf68741c\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.434347687Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.44256073Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"uri":"/crucible/0/regions/a165be3c-2dc9-4e51-a247-96660f2a91ef/snapshots/bd357839-a321-4a4a-b806-b5f3cf68741c","method":"GET","req_id":"36bea492-1401-4465-806b-76f8c424b467","remote_addr":"[fd00:1122:3344:101::a]:40440","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":27740,"response_code":"200"}
{"msg":"path is \"/data/regions/a165be3c-2dc9-4e51-a247-96660f2a91ef\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.442783096Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"datafile"}
{"msg":"dataset is oxp_f8b5491e-78ae-4725-a4e0-5d2f23407448/crucible/regions/a165be3c-2dc9-4e51-a247-96660f2a91ef","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.452234573Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"datafile"}
{"msg":"snapshot is oxp_f8b5491e-78ae-4725-a4e0-5d2f23407448/crucible/regions/a165be3c-2dc9-4e51-a247-96660f2a91ef@bd357839-a321-4a4a-b806-b5f3cf68741c","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.460727694Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_f8b5491e-78ae-4725-a4e0-5d2f23407448/crucible/regions/a165be3c-2dc9-4e51-a247-96660f2a91ef\", \"bd357839-a321-4a4a-b806-b5f3cf68741c\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.460757544Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot a165be3c-2dc9-4e51-a247-96660f2a91ef-bd357839-a321-4a4a-b806-b5f3cf68741c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.473975283Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"datafile"}
{"msg":"applying SMF actions for region a165be3c-2dc9-4e51-a247-96660f2a91ef running snapshot bd357839-a321-4a4a-b806-b5f3cf68741c (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.474126477Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.474175048Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"uri":"/crucible/0/regions/a165be3c-2dc9-4e51-a247-96660f2a91ef/snapshots/bd357839-a321-4a4a-b806-b5f3cf68741c/run","method":"POST","req_id":"0cf50ea0-8c40-4fbd-9c1e-e584306b0efa","remote_addr":"[fd00:1122:3344:101::a]:40440","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":31415,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-a165be3c-2dc9-4e51-a247-96660f2a91ef-bd357839-a321-4a4a-b806-b5f3cf68741c","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.476894612Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-a165be3c-2dc9-4e51-a247-96660f2a91ef-bd357839-a321-4a4a-b806-b5f3cf68741c","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.483617855Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-a165be3c-2dc9-4e51-a247-96660f2a91ef-bd357839-a321-4a4a-b806-b5f3cf68741c","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.495889418Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a165be3c-2dc9-4e51-a247-96660f2a91ef/.zfs/snapshot/bd357839-a321-4a4a-b806-b5f3cf68741c","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.496109554Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.496174196Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.496232737Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.496281499Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.49634658Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.500667698Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.504715347Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"worker"}
{"msg":"region a165be3c-2dc9-4e51-a247-96660f2a91ef running snapshot bd357839-a321-4a4a-b806-b5f3cf68741c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-27T21:56:38.504728008Z","hostname":"oxz_crucible_9ef26a0e-ea1c-4ec7-8e58-ecb23473ac6f","pid":21103,"component":"datafile"}