{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:14.873188349Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:64083"}
{"msg":"region 59a4ce8d-08da-4013-b9b7-a5dcd19d996d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:14.932899876Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:14.933808636Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"uri":"/crucible/0/regions","method":"POST","req_id":"0c29104e-ad55-48a0-bc6e-9c8f4a7f1db3","remote_addr":"[fd00:1122:3344:101::a]:64083","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":995,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:14.933824466Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_5e7d7eb6-810f-4af3-87ba-75bd65a1dd51/crucible/regions/59a4ce8d-08da-4013-b9b7-a5dcd19d996d","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:14.945938556Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_5e7d7eb6-810f-4af3-87ba-75bd65a1dd51/crucible/regions/59a4ce8d-08da-4013-b9b7-a5dcd19d996d","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:14.945969696Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"59a4ce8d-08da-4013-b9b7-a5dcd19d996d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/59a4ce8d-08da-4013-b9b7-a5dcd19d996d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:14.997419937Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"region":"59a4ce8d-08da-4013-b9b7-a5dcd19d996d","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"59a4ce8d-08da-4013-b9b7-a5dcd19d996d\" \"--data\" \"/data/regions/59a4ce8d-08da-4013-b9b7-a5dcd19d996d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:14.997468348Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"region":"59a4ce8d-08da-4013-b9b7-a5dcd19d996d","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.126087697Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"region":"59a4ce8d-08da-4013-b9b7-a5dcd19d996d","component":"worker"}
{"msg":"region 59a4ce8d-08da-4013-b9b7-a5dcd19d996d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.126147328Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.126248359Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-59a4ce8d-08da-4013-b9b7-a5dcd19d996d","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.126731895Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-59a4ce8d-08da-4013-b9b7-a5dcd19d996d","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.131960415Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.137875573Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"uri":"/crucible/0/regions","method":"POST","req_id":"13c68d54-e5c7-4361-b65c-63588fad2e10","remote_addr":"[fd00:1122:3344:101::a]:64083","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":112,"response_code":"200"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.146408741Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-59a4ce8d-08da-4013-b9b7-a5dcd19d996d","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.150286676Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/59a4ce8d-08da-4013-b9b7-a5dcd19d996d","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.15065899Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.150724711Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.150786501Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.150837782Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.154586735Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/a42dae0c-dbd9-42f9-a452-8df3a164f048","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.160150509Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.160257721Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.160364362Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.160376482Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.160429303Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a42dae0c-dbd9-42f9-a452-8df3a164f048","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.160490163Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a42dae0c-dbd9-42f9-a452-8df3a164f048","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.160583034Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.160652535Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.160708016Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.160767207Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.172160027Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.172713014Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.763613912Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:62114"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.767139393Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"uri":"/crucible/0/regions/a42dae0c-dbd9-42f9-a452-8df3a164f048","method":"GET","req_id":"2ae5d9b9-174e-44df-a229-3738b318c04f","remote_addr":"[fd00:1122:3344:101::a]:62114","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":3502,"response_code":"200"}
{"msg":"path is \"/data/regions/a42dae0c-dbd9-42f9-a452-8df3a164f048\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.767393126Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"datafile"}
{"msg":"dataset is oxp_5e7d7eb6-810f-4af3-87ba-75bd65a1dd51/crucible/regions/a42dae0c-dbd9-42f9-a452-8df3a164f048","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.776057206Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"datafile"}
{"msg":"snapshot is oxp_5e7d7eb6-810f-4af3-87ba-75bd65a1dd51/crucible/regions/a42dae0c-dbd9-42f9-a452-8df3a164f048@57b539da-ef47-4db6-9af6-4be24813fd01","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.783898876Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_5e7d7eb6-810f-4af3-87ba-75bd65a1dd51/crucible/regions/a42dae0c-dbd9-42f9-a452-8df3a164f048\", \"57b539da-ef47-4db6-9af6-4be24813fd01\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.783931716Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.796719064Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"uri":"/crucible/0/regions/a42dae0c-dbd9-42f9-a452-8df3a164f048/snapshots/57b539da-ef47-4db6-9af6-4be24813fd01","method":"GET","req_id":"5dc109e8-c0f2-4f5f-b1fd-df3ac5aa6a14","remote_addr":"[fd00:1122:3344:101::a]:62114","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":29382,"response_code":"200"}
{"msg":"path is \"/data/regions/a42dae0c-dbd9-42f9-a452-8df3a164f048\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.796893735Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"datafile"}
{"msg":"dataset is oxp_5e7d7eb6-810f-4af3-87ba-75bd65a1dd51/crucible/regions/a42dae0c-dbd9-42f9-a452-8df3a164f048","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.805514615Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"datafile"}
{"msg":"snapshot is oxp_5e7d7eb6-810f-4af3-87ba-75bd65a1dd51/crucible/regions/a42dae0c-dbd9-42f9-a452-8df3a164f048@57b539da-ef47-4db6-9af6-4be24813fd01","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.814610929Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_5e7d7eb6-810f-4af3-87ba-75bd65a1dd51/crucible/regions/a42dae0c-dbd9-42f9-a452-8df3a164f048\", \"57b539da-ef47-4db6-9af6-4be24813fd01\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.81465128Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot a42dae0c-dbd9-42f9-a452-8df3a164f048-57b539da-ef47-4db6-9af6-4be24813fd01 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.823628623Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"datafile"}
{"msg":"applying SMF actions for region a42dae0c-dbd9-42f9-a452-8df3a164f048 running snapshot 57b539da-ef47-4db6-9af6-4be24813fd01 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.823816865Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.823847896Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"uri":"/crucible/0/regions/a42dae0c-dbd9-42f9-a452-8df3a164f048/snapshots/57b539da-ef47-4db6-9af6-4be24813fd01/run","method":"POST","req_id":"72ae1744-ebdf-476d-9244-05ffadbe2c93","remote_addr":"[fd00:1122:3344:101::a]:62114","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":26985,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/a42dae0c-dbd9-42f9-a452-8df3a164f048","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.825401974Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.825490835Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.825576476Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.825598246Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.825625936Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a42dae0c-dbd9-42f9-a452-8df3a164f048","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.825665086Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a42dae0c-dbd9-42f9-a452-8df3a164f048","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.82597371Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.826030571Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.826077751Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.826122492Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.829989396Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-a42dae0c-dbd9-42f9-a452-8df3a164f048-57b539da-ef47-4db6-9af6-4be24813fd01","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.830429971Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-a42dae0c-dbd9-42f9-a452-8df3a164f048-57b539da-ef47-4db6-9af6-4be24813fd01","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.83723389Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-a42dae0c-dbd9-42f9-a452-8df3a164f048-57b539da-ef47-4db6-9af6-4be24813fd01","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.849142567Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a42dae0c-dbd9-42f9-a452-8df3a164f048/.zfs/snapshot/57b539da-ef47-4db6-9af6-4be24813fd01","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.849364139Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.84941627Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.84945704Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.849494741Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.849540331Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.853281694Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.857619374Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"region a42dae0c-dbd9-42f9-a452-8df3a164f048 running snapshot 57b539da-ef47-4db6-9af6-4be24813fd01 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:15.857633234Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.727522392Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:47608"}
{"msg":"region e1ea3c1a-8ddf-4274-8a3f-e615a628ce84 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.786985736Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.787223149Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.787240639Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"uri":"/crucible/0/regions","method":"POST","req_id":"29b68828-ca63-414a-910a-7a7b083da42c","remote_addr":"[fd00:1122:3344:101::a]:47608","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":307,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_5e7d7eb6-810f-4af3-87ba-75bd65a1dd51/crucible/regions/e1ea3c1a-8ddf-4274-8a3f-e615a628ce84","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.797253724Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_5e7d7eb6-810f-4af3-87ba-75bd65a1dd51/crucible/regions/e1ea3c1a-8ddf-4274-8a3f-e615a628ce84","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.797288594Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e1ea3c1a-8ddf-4274-8a3f-e615a628ce84\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e1ea3c1a-8ddf-4274-8a3f-e615a628ce84\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.844984133Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"region":"e1ea3c1a-8ddf-4274-8a3f-e615a628ce84","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e1ea3c1a-8ddf-4274-8a3f-e615a628ce84\" \"--data\" \"/data/regions/e1ea3c1a-8ddf-4274-8a3f-e615a628ce84\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.846783304Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"region":"e1ea3c1a-8ddf-4274-8a3f-e615a628ce84","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.910494217Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"region":"e1ea3c1a-8ddf-4274-8a3f-e615a628ce84","component":"worker"}
{"msg":"region e1ea3c1a-8ddf-4274-8a3f-e615a628ce84 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.910528837Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.910639298Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/a42dae0c-dbd9-42f9-a452-8df3a164f048","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.912014354Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.912104495Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.912186136Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.912192567Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.912234307Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a42dae0c-dbd9-42f9-a452-8df3a164f048","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.912278937Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a42dae0c-dbd9-42f9-a452-8df3a164f048","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.912658102Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.912719073Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.912767223Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.912812344Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.91683678Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e1ea3c1a-8ddf-4274-8a3f-e615a628ce84","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.917302065Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e1ea3c1a-8ddf-4274-8a3f-e615a628ce84","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.920957407Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.928308662Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e1ea3c1a-8ddf-4274-8a3f-e615a628ce84","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.93071874Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e1ea3c1a-8ddf-4274-8a3f-e615a628ce84","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.930921872Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.930981023Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.931022403Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.931061904Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.934160889Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:18.93948565Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:19.053464881Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"uri":"/crucible/0/regions","method":"POST","req_id":"fa210bf7-115b-4c3e-8821-0fdab99e58d0","remote_addr":"[fd00:1122:3344:101::a]:47608","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":82,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:42.778614915Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:46408"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:42.778708126Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"uri":"/crucible/0/regions/e1ea3c1a-8ddf-4274-8a3f-e615a628ce84","method":"GET","req_id":"769cd3a3-6236-4372-b5d0-ec6dfe51e050","remote_addr":"[fd00:1122:3344:101::a]:46408","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":54,"response_code":"200"}
{"msg":"path is \"/data/regions/e1ea3c1a-8ddf-4274-8a3f-e615a628ce84\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:42.778981119Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"datafile"}
{"msg":"dataset is oxp_5e7d7eb6-810f-4af3-87ba-75bd65a1dd51/crucible/regions/e1ea3c1a-8ddf-4274-8a3f-e615a628ce84","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:42.790111787Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"datafile"}
{"msg":"region e1ea3c1a-8ddf-4274-8a3f-e615a628ce84 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:42.800135053Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:42.800358955Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:42.802791053Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"uri":"/crucible/0/regions/e1ea3c1a-8ddf-4274-8a3f-e615a628ce84","method":"DELETE","req_id":"0e7259c2-a604-4a28-a3a2-fb8fa2b21fa4","remote_addr":"[fd00:1122:3344:101::a]:46408","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":23849,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:42.802991675Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"uri":"/crucible/0/regions/e1ea3c1a-8ddf-4274-8a3f-e615a628ce84","method":"GET","req_id":"f8069ca4-c3f6-4eda-86e3-a8657f036df6","remote_addr":"[fd00:1122:3344:101::a]:46408","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":28,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-e1ea3c1a-8ddf-4274-8a3f-e615a628ce84 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:42.803831755Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/a42dae0c-dbd9-42f9-a452-8df3a164f048","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:42.809064125Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:42.809144446Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:42.809223767Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:42.809235387Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:42.809276768Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a42dae0c-dbd9-42f9-a452-8df3a164f048","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:42.809344708Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a42dae0c-dbd9-42f9-a452-8df3a164f048","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:42.809660522Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:42.809727923Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:42.809776093Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:42.809823924Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:42.813328784Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:42.81469616Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_5e7d7eb6-810f-4af3-87ba-75bd65a1dd51/crucible/regions/e1ea3c1a-8ddf-4274-8a3f-e615a628ce84\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:42.825034749Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"region":"e1ea3c1a-8ddf-4274-8a3f-e615a628ce84","component":"worker"}
{"msg":"region e1ea3c1a-8ddf-4274-8a3f-e615a628ce84 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:42.869629622Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:34:42.953361225Z","hostname":"oxz_crucible_111506b1-d772-40cc-bc9e-e504e433bf72","pid":22848,"uri":"/crucible/0/regions/e1ea3c1a-8ddf-4274-8a3f-e615a628ce84","method":"GET","req_id":"f0ffe60b-3337-4582-8518-f55847483ca7","remote_addr":"[fd00:1122:3344:101::a]:46408","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":91,"response_code":"200"}