[ May  8 06:16:50 Enabled. ]
[ May  8 06:16:50 Rereading configuration. ]
[ May  8 06:16:52 Rereading configuration. ]
[ May  8 06:16:57 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_372a7831-84b0-4e2e-a46a-0709dcd2e0f9/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_372a7831-84b0-4e2e-a46a-0709dcd2e0f9/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:16:57.215260197Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:16:57.221278491Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:16:57.221287822Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:16:57.236483705Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:16:57.31568759Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:46.938367932Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:43196"}
{"msg":"region 14dd8d6d-0074-4c46-8740-77d6750a8efd state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:46.972944558Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:46.973571967Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"uri":"/crucible/0/regions","method":"POST","req_id":"969de956-462c-4a0a-903d-260c6d992aa9","remote_addr":"[fd00:1122:3344:101::b]:43196","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":15538,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:46.976129103Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_372a7831-84b0-4e2e-a46a-0709dcd2e0f9/crucible/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:46.985361403Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_372a7831-84b0-4e2e-a46a-0709dcd2e0f9/crucible/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:46.985392053Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"14dd8d6d-0074-4c46-8740-77d6750a8efd\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.034555045Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"region":"14dd8d6d-0074-4c46-8740-77d6750a8efd","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"14dd8d6d-0074-4c46-8740-77d6750a8efd\" \"--data\" \"/data/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.034607036Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"region":"14dd8d6d-0074-4c46-8740-77d6750a8efd","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.139466451Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"uri":"/crucible/0/regions","method":"POST","req_id":"36624ca4-dd0e-4726-938e-961411969808","remote_addr":"[fd00:1122:3344:101::b]:43196","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":71,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.369428718Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"region":"14dd8d6d-0074-4c46-8740-77d6750a8efd","component":"worker"}
{"msg":"region 14dd8d6d-0074-4c46-8740-77d6750a8efd state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.369476028Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.36959223Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-14dd8d6d-0074-4c46-8740-77d6750a8efd","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.370063097Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-14dd8d6d-0074-4c46-8740-77d6750a8efd","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.376661729Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.398143322Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-14dd8d6d-0074-4c46-8740-77d6750a8efd","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.401000742Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.401747873Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.403819752Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.403866772Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.403907553Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.412400472Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.418841183Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.781161322Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"uri":"/crucible/0/regions","method":"POST","req_id":"6b24116b-a956-4bf8-ac7d-5f0b79db9fd1","remote_addr":"[fd00:1122:3344:101::b]:43196","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":53,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.910249159Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:40972"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.910916099Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"uri":"/crucible/0/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd","method":"GET","req_id":"61cea81f-b476-4a7a-9279-d8140619dec7","remote_addr":"[fd00:1122:3344:101::b]:40972","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":53,"response_code":"200"}
{"msg":"path is \"/data/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.913001708Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"datafile"}
{"msg":"dataset is oxp_372a7831-84b0-4e2e-a46a-0709dcd2e0f9/crucible/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.922215618Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"datafile"}
{"msg":"snapshot is oxp_372a7831-84b0-4e2e-a46a-0709dcd2e0f9/crucible/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd@32c9588a-9338-4f7d-8835-b51c992dbfc6","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.930670377Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_372a7831-84b0-4e2e-a46a-0709dcd2e0f9/crucible/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd\", \"32c9588a-9338-4f7d-8835-b51c992dbfc6\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.930706587Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.942516184Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"uri":"/crucible/0/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd/snapshots/32c9588a-9338-4f7d-8835-b51c992dbfc6","method":"GET","req_id":"76a5c51c-6378-47d6-9b6b-a55b184307c1","remote_addr":"[fd00:1122:3344:101::b]:40972","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":29516,"response_code":"200"}
{"msg":"path is \"/data/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.945519156Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"datafile"}
{"msg":"dataset is oxp_372a7831-84b0-4e2e-a46a-0709dcd2e0f9/crucible/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.954916858Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"datafile"}
{"msg":"snapshot is oxp_372a7831-84b0-4e2e-a46a-0709dcd2e0f9/crucible/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd@32c9588a-9338-4f7d-8835-b51c992dbfc6","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.962970581Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_372a7831-84b0-4e2e-a46a-0709dcd2e0f9/crucible/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd\", \"32c9588a-9338-4f7d-8835-b51c992dbfc6\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.963002572Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 14dd8d6d-0074-4c46-8740-77d6750a8efd-32c9588a-9338-4f7d-8835-b51c992dbfc6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.972762629Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"datafile"}
{"msg":"applying SMF actions for region 14dd8d6d-0074-4c46-8740-77d6750a8efd running snapshot 32c9588a-9338-4f7d-8835-b51c992dbfc6 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.973397148Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.973430628Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"uri":"/crucible/0/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd/snapshots/32c9588a-9338-4f7d-8835-b51c992dbfc6/run","method":"POST","req_id":"106bf566-75cc-4525-9e04-f0e3a8429a81","remote_addr":"[fd00:1122:3344:101::b]:40972","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":27928,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.990184704Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.990271456Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.990353157Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.990361477Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.990404857Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-14dd8d6d-0074-4c46-8740-77d6750a8efd","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.990453648Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.99055618Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.99062066Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.990669931Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.990718282Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.000700772Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-14dd8d6d-0074-4c46-8740-77d6750a8efd-32c9588a-9338-4f7d-8835-b51c992dbfc6","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.00123791Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-14dd8d6d-0074-4c46-8740-77d6750a8efd-32c9588a-9338-4f7d-8835-b51c992dbfc6","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.004238202Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-14dd8d6d-0074-4c46-8740-77d6750a8efd-32c9588a-9338-4f7d-8835-b51c992dbfc6","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.016002927Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd/.zfs/snapshot/32c9588a-9338-4f7d-8835-b51c992dbfc6","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.01620387Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.016258661Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.016301502Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.016341162Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.016390153Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.019666719Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.024269644Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"region 14dd8d6d-0074-4c46-8740-77d6750a8efd running snapshot 32c9588a-9338-4f7d-8835-b51c992dbfc6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.024287274Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"datafile"}
{"msg":"path is \"/data/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.131245969Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"datafile"}
{"msg":"dataset is oxp_372a7831-84b0-4e2e-a46a-0709dcd2e0f9/crucible/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.140395568Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"datafile"}
{"msg":"snapshot is oxp_372a7831-84b0-4e2e-a46a-0709dcd2e0f9/crucible/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd@32c9588a-9338-4f7d-8835-b51c992dbfc6","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.14980743Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_372a7831-84b0-4e2e-a46a-0709dcd2e0f9/crucible/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd\", \"32c9588a-9338-4f7d-8835-b51c992dbfc6\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.149854871Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.158436912Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"uri":"/crucible/0/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd/snapshots/32c9588a-9338-4f7d-8835-b51c992dbfc6/run","method":"POST","req_id":"e08b06d3-5771-4495-815a-58a708193e17","remote_addr":"[fd00:1122:3344:101::b]:40972","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":27197,"response_code":"200"}
{"msg":"region b05c5459-f557-4383-b001-ec73ea940de9 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:48.908733617Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:48.90893053Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"uri":"/crucible/0/regions","method":"POST","req_id":"8fa79b92-c82b-4c23-995e-f9383c6b9c84","remote_addr":"[fd00:1122:3344:101::b]:40972","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":255,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:48.90897429Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_372a7831-84b0-4e2e-a46a-0709dcd2e0f9/crucible/regions/b05c5459-f557-4383-b001-ec73ea940de9","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:48.91888769Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_372a7831-84b0-4e2e-a46a-0709dcd2e0f9/crucible/regions/b05c5459-f557-4383-b001-ec73ea940de9","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:48.91891291Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b05c5459-f557-4383-b001-ec73ea940de9\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b05c5459-f557-4383-b001-ec73ea940de9\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:48.974284939Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"region":"b05c5459-f557-4383-b001-ec73ea940de9","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b05c5459-f557-4383-b001-ec73ea940de9\" \"--data\" \"/data/regions/b05c5459-f557-4383-b001-ec73ea940de9\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:48.976120905Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"region":"b05c5459-f557-4383-b001-ec73ea940de9","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.05036497Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"region":"b05c5459-f557-4383-b001-ec73ea940de9","component":"worker"}
{"msg":"region b05c5459-f557-4383-b001-ec73ea940de9 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.05039636Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.050510592Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.05106692Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.051149591Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.051228132Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.051240812Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.051284383Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-14dd8d6d-0074-4c46-8740-77d6750a8efd","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.051331174Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.051680388Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.051744599Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.05179108Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.051837211Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.056273933Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b05c5459-f557-4383-b001-ec73ea940de9","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.05674161Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b05c5459-f557-4383-b001-ec73ea940de9","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.067190877Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.077232488Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b05c5459-f557-4383-b001-ec73ea940de9","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.080429353Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b05c5459-f557-4383-b001-ec73ea940de9","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.080632916Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.080694887Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.080735897Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.080776358Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.084413379Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.089962767Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.181083309Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"uri":"/crucible/0/regions","method":"POST","req_id":"3a7f5652-019a-4510-8d24-9098c4658b93","remote_addr":"[fd00:1122:3344:101::b]:40972","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":57,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.436887924Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"uri":"/crucible/0/regions/b05c5459-f557-4383-b001-ec73ea940de9","method":"GET","req_id":"10242121-e2f1-4fc3-a2d4-bd7a2876e584","remote_addr":"[fd00:1122:3344:101::b]:40972","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":52,"response_code":"200"}
{"msg":"path is \"/data/regions/b05c5459-f557-4383-b001-ec73ea940de9\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.440955402Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"datafile"}
{"msg":"dataset is oxp_372a7831-84b0-4e2e-a46a-0709dcd2e0f9/crucible/regions/b05c5459-f557-4383-b001-ec73ea940de9","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.454795056Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"datafile"}
{"msg":"region b05c5459-f557-4383-b001-ec73ea940de9 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.466060505Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.466239188Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.466285888Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"uri":"/crucible/0/regions/b05c5459-f557-4383-b001-ec73ea940de9","method":"DELETE","req_id":"9e9d15cf-846a-4813-bb85-6849d0f78fd6","remote_addr":"[fd00:1122:3344:101::b]:40972","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":25406,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-b05c5459-f557-4383-b001-ec73ea940de9 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.467550496Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.469534114Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"uri":"/crucible/0/regions/b05c5459-f557-4383-b001-ec73ea940de9","method":"GET","req_id":"d0477b58-ebae-46cf-9fb8-04a61031e597","remote_addr":"[fd00:1122:3344:101::b]:40972","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":40,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.475593269Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.475726431Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.475823082Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.475871643Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.475878833Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-14dd8d6d-0074-4c46-8740-77d6750a8efd","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.475932794Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/14dd8d6d-0074-4c46-8740-77d6750a8efd","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.476281439Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.47636561Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.476423011Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.476476061Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.480393477Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.481757946Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_372a7831-84b0-4e2e-a46a-0709dcd2e0f9/crucible/regions/b05c5459-f557-4383-b001-ec73ea940de9\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.49339329Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"region":"b05c5459-f557-4383-b001-ec73ea940de9","component":"worker"}
{"msg":"region b05c5459-f557-4383-b001-ec73ea940de9 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.535695325Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.56020884Z","hostname":"oxz_crucible_469bd51b-2ce8-43ba-b3fb-4166cf732d67","pid":25995,"uri":"/crucible/0/regions/b05c5459-f557-4383-b001-ec73ea940de9","method":"GET","req_id":"1f4ea13f-1d0d-4efb-ad39-8c157b47bb23","remote_addr":"[fd00:1122:3344:101::b]:40972","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":63,"response_code":"200"}