[ Jan  9 01:33:35 Enabled. ]
[ Jan  9 01:33:35 Rereading configuration. ]
[ Jan  9 01:33:37 Rereading configuration. ]
[ Jan  9 01:33:40 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_9979ed37-d6a2-48bf-a63c-5d900fb82be1/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_9979ed37-d6a2-48bf-a63c-5d900fb82be1/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:33:40.996577482Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:33:41.000215231Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:33:41.000230831Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:33:41.016627395Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:33:41.099450757Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:36.651443527Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:55460"}
{"msg":"region c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:36.684756443Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:36.689377958Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:36.694246996Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"uri":"/crucible/0/regions","method":"POST","req_id":"364eea99-1ca2-4862-bd76-dc7460b6368e","remote_addr":"[fd00:1122:3344:101::a]:55460","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":15295,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_9979ed37-d6a2-48bf-a63c-5d900fb82be1/crucible/regions/c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:36.701010045Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_9979ed37-d6a2-48bf-a63c-5d900fb82be1/crucible/regions/c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:36.701037215Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:36.768908187Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"region":"c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef\" \"--data\" \"/data/regions/c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:36.770920789Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"region":"c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:36.890265279Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"region":"c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef","component":"worker"}
{"msg":"region c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:36.89031964Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:36.890412831Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:36.890887429Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:36.897269992Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:36.91209276Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:36.916138175Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:36.917496877Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:36.917580738Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:36.917625429Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:36.91766427Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:36.922813033Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:36.929786215Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:37.038997811Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"uri":"/crucible/0/regions","method":"POST","req_id":"5c8c5c1a-dfb1-437e-92ef-4bdbe63a117a","remote_addr":"[fd00:1122:3344:101::a]:55460","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":66,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:37.36096733Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"uri":"/crucible/0/regions/c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef","method":"GET","req_id":"14e638e7-5b91-4f66-a087-635d7526a72d","remote_addr":"[fd00:1122:3344:101::a]:55460","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":89,"response_code":"200"}
{"msg":"path is \"/data/regions/c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:37.364906274Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"datafile"}
{"msg":"dataset is oxp_9979ed37-d6a2-48bf-a63c-5d900fb82be1/crucible/regions/c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:37.376315447Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"datafile"}
{"msg":"region c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:37.385767149Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:37.385900242Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:37.385958263Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"uri":"/crucible/0/regions/c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef","method":"DELETE","req_id":"bedc787e-1f8a-4b79-8195-ca0eb596842c","remote_addr":"[fd00:1122:3344:101::a]:55460","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":21093,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:37.3889403Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:37.389246295Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"uri":"/crucible/0/regions/c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef","method":"GET","req_id":"08bdbd41-bd50-45ef-89a8-e6f1e318ea1f","remote_addr":"[fd00:1122:3344:101::a]:55460","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":34,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:37.393333131Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_9979ed37-d6a2-48bf-a63c-5d900fb82be1/crucible/regions/c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:37.404964588Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"region":"c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef","component":"worker"}
{"msg":"region c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:37.476857585Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:37.523068448Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"uri":"/crucible/0/regions/c4e90381-5ed4-4b11-a3d8-bf664ecfe3ef","method":"GET","req_id":"28181e26-e29f-4578-b560-d6eb31d8848a","remote_addr":"[fd00:1122:3344:101::a]:55460","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":82,"response_code":"200"}
{"msg":"region 06123c95-062d-4dc1-bb1f-28631f0c565c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:39.390965135Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:39.391099027Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:39.391118277Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"uri":"/crucible/0/regions","method":"POST","req_id":"09fa6037-2da6-46ad-91e8-212eb15b3931","remote_addr":"[fd00:1122:3344:101::a]:55460","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":213,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_9979ed37-d6a2-48bf-a63c-5d900fb82be1/crucible/regions/06123c95-062d-4dc1-bb1f-28631f0c565c","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:39.399090806Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_9979ed37-d6a2-48bf-a63c-5d900fb82be1/crucible/regions/06123c95-062d-4dc1-bb1f-28631f0c565c","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:39.399126166Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"06123c95-062d-4dc1-bb1f-28631f0c565c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/06123c95-062d-4dc1-bb1f-28631f0c565c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:39.461988277Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"region":"06123c95-062d-4dc1-bb1f-28631f0c565c","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"06123c95-062d-4dc1-bb1f-28631f0c565c\" \"--data\" \"/data/regions/06123c95-062d-4dc1-bb1f-28631f0c565c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:39.462047688Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"region":"06123c95-062d-4dc1-bb1f-28631f0c565c","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:39.549364943Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"uri":"/crucible/0/regions","method":"POST","req_id":"55a62372-4e34-4f85-a511-79c6c13c33df","remote_addr":"[fd00:1122:3344:101::a]:55460","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":94,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:39.553951867Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"region":"06123c95-062d-4dc1-bb1f-28631f0c565c","component":"worker"}
{"msg":"region 06123c95-062d-4dc1-bb1f-28631f0c565c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:39.553984867Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:39.554082769Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-06123c95-062d-4dc1-bb1f-28631f0c565c","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:39.555475921Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-06123c95-062d-4dc1-bb1f-28631f0c565c","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:39.559073469Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:39.566882395Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-06123c95-062d-4dc1-bb1f-28631f0c565c","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:39.570636215Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/06123c95-062d-4dc1-bb1f-28631f0c565c","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:39.57093065Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:39.570988211Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:39.571033412Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:39.571078092Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:39.575401352Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:39.580812859Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:34:40.067710281Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"uri":"/crucible/0/regions","method":"POST","req_id":"8ed2cdea-74e7-4c32-80b2-dcdc04ed2c69","remote_addr":"[fd00:1122:3344:101::a]:55460","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":57,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:51.804146575Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:36392"}
{"msg":"region 8a4367e8-49d8-4093-8eee-595eb1f70464 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:51.804275997Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:51.80510764Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:51.805124201Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"uri":"/crucible/0/regions","method":"POST","req_id":"e02f3414-cc4b-456c-b504-3d7b249f6ee5","remote_addr":"[fd00:1122:3344:101::a]:36392","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":913,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_9979ed37-d6a2-48bf-a63c-5d900fb82be1/crucible/regions/8a4367e8-49d8-4093-8eee-595eb1f70464","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:51.815093391Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_9979ed37-d6a2-48bf-a63c-5d900fb82be1/crucible/regions/8a4367e8-49d8-4093-8eee-595eb1f70464","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:51.815131672Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"8a4367e8-49d8-4093-8eee-595eb1f70464\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/8a4367e8-49d8-4093-8eee-595eb1f70464\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:51.871091252Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"region":"8a4367e8-49d8-4093-8eee-595eb1f70464","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"8a4367e8-49d8-4093-8eee-595eb1f70464\" \"--data\" \"/data/regions/8a4367e8-49d8-4093-8eee-595eb1f70464\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:51.871144623Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"region":"8a4367e8-49d8-4093-8eee-595eb1f70464","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:51.952713045Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"region":"8a4367e8-49d8-4093-8eee-595eb1f70464","component":"worker"}
{"msg":"region 8a4367e8-49d8-4093-8eee-595eb1f70464 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:51.952757595Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:51.952856937Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-8a4367e8-49d8-4093-8eee-595eb1f70464","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:51.960096873Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-8a4367e8-49d8-4093-8eee-595eb1f70464","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:51.965319317Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:51.97480479Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-8a4367e8-49d8-4093-8eee-595eb1f70464","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:51.978434648Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8a4367e8-49d8-4093-8eee-595eb1f70464","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:51.978706013Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:51.978769794Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:51.978815735Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:51.978863145Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:51.982498274Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:51.986985646Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:52.152551119Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"uri":"/crucible/0/regions","method":"POST","req_id":"74b62988-8e37-403d-a08c-d9edc1e80ea8","remote_addr":"[fd00:1122:3344:101::a]:36392","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:53.752027748Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"uri":"/crucible/0/regions/06123c95-062d-4dc1-bb1f-28631f0c565c","method":"GET","req_id":"e29db890-15f3-4637-aa9b-84abc363c693","remote_addr":"[fd00:1122:3344:101::a]:36392","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"path is \"/data/regions/06123c95-062d-4dc1-bb1f-28631f0c565c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:53.758072646Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"datafile"}
{"msg":"dataset is oxp_9979ed37-d6a2-48bf-a63c-5d900fb82be1/crucible/regions/06123c95-062d-4dc1-bb1f-28631f0c565c","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:53.771323469Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"datafile"}
{"msg":"snapshot is oxp_9979ed37-d6a2-48bf-a63c-5d900fb82be1/crucible/regions/06123c95-062d-4dc1-bb1f-28631f0c565c@841ec45a-7939-4ef0-99e1-ca849899fc25","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:53.783046467Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_9979ed37-d6a2-48bf-a63c-5d900fb82be1/crucible/regions/06123c95-062d-4dc1-bb1f-28631f0c565c\", \"841ec45a-7939-4ef0-99e1-ca849899fc25\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:53.783103628Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:53.798398084Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"uri":"/crucible/0/regions/06123c95-062d-4dc1-bb1f-28631f0c565c/snapshots/841ec45a-7939-4ef0-99e1-ca849899fc25","method":"GET","req_id":"3313d156-413a-4f2a-b65d-b826df108281","remote_addr":"[fd00:1122:3344:101::a]:36392","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":41175,"response_code":"200"}
{"msg":"path is \"/data/regions/06123c95-062d-4dc1-bb1f-28631f0c565c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:53.804272409Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"datafile"}
{"msg":"dataset is oxp_9979ed37-d6a2-48bf-a63c-5d900fb82be1/crucible/regions/06123c95-062d-4dc1-bb1f-28631f0c565c","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:53.816057928Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"datafile"}
{"msg":"snapshot is oxp_9979ed37-d6a2-48bf-a63c-5d900fb82be1/crucible/regions/06123c95-062d-4dc1-bb1f-28631f0c565c@841ec45a-7939-4ef0-99e1-ca849899fc25","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:53.829017207Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_9979ed37-d6a2-48bf-a63c-5d900fb82be1/crucible/regions/06123c95-062d-4dc1-bb1f-28631f0c565c\", \"841ec45a-7939-4ef0-99e1-ca849899fc25\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:53.829061508Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 06123c95-062d-4dc1-bb1f-28631f0c565c-841ec45a-7939-4ef0-99e1-ca849899fc25 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:53.841355845Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"datafile"}
{"msg":"applying SMF actions for region 06123c95-062d-4dc1-bb1f-28631f0c565c running snapshot 841ec45a-7939-4ef0-99e1-ca849899fc25 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:53.841601559Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:53.84163791Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"uri":"/crucible/0/regions/06123c95-062d-4dc1-bb1f-28631f0c565c/snapshots/841ec45a-7939-4ef0-99e1-ca849899fc25/run","method":"POST","req_id":"14e5dd17-b1cd-44e2-a0ab-566ef12b3bab","remote_addr":"[fd00:1122:3344:101::a]:36392","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":37417,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-06123c95-062d-4dc1-bb1f-28631f0c565c-841ec45a-7939-4ef0-99e1-ca849899fc25","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:53.846043451Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-06123c95-062d-4dc1-bb1f-28631f0c565c-841ec45a-7939-4ef0-99e1-ca849899fc25","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:53.849832171Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-06123c95-062d-4dc1-bb1f-28631f0c565c-841ec45a-7939-4ef0-99e1-ca849899fc25","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:53.864568639Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/06123c95-062d-4dc1-bb1f-28631f0c565c/.zfs/snapshot/841ec45a-7939-4ef0-99e1-ca849899fc25","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:53.865025546Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:53.865109177Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:53.865189949Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:53.86525762Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:53.865353101Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:53.871057753Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:53.876644043Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"worker"}
{"msg":"region 06123c95-062d-4dc1-bb1f-28631f0c565c running snapshot 841ec45a-7939-4ef0-99e1-ca849899fc25 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:53.876664223Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"datafile"}
{"msg":"path is \"/data/regions/06123c95-062d-4dc1-bb1f-28631f0c565c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:54.067021365Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"datafile"}
{"msg":"dataset is oxp_9979ed37-d6a2-48bf-a63c-5d900fb82be1/crucible/regions/06123c95-062d-4dc1-bb1f-28631f0c565c","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:54.075785296Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"datafile"}
{"msg":"snapshot is oxp_9979ed37-d6a2-48bf-a63c-5d900fb82be1/crucible/regions/06123c95-062d-4dc1-bb1f-28631f0c565c@841ec45a-7939-4ef0-99e1-ca849899fc25","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:54.084466786Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_9979ed37-d6a2-48bf-a63c-5d900fb82be1/crucible/regions/06123c95-062d-4dc1-bb1f-28631f0c565c\", \"841ec45a-7939-4ef0-99e1-ca849899fc25\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:54.084503457Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:44:54.091797524Z","hostname":"oxz_crucible_fb10fea9-36d5-42a4-9bd1-be85290923fe","pid":23373,"uri":"/crucible/0/regions/06123c95-062d-4dc1-bb1f-28631f0c565c/snapshots/841ec45a-7939-4ef0-99e1-ca849899fc25/run","method":"POST","req_id":"bd7829b4-452c-499a-a9c7-4c2a571dfc1d","remote_addr":"[fd00:1122:3344:101::a]:36392","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":24834,"response_code":"200"}