[ Aug 23 04:22:30 Enabled. ]
[ Aug 23 04:22:30 Rereading configuration. ]
[ Aug 23 04:22:31 Rereading configuration. ]
[ Aug 23 04:22:33 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_3902ae36-9ef8-49e4-a803-969ef89cd471/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_3902ae36-9ef8-49e4-a803-969ef89cd471/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:22:33.823326429Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:22:33.82425059Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:22:33.824260521Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:22:33.836780434Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:22:34.346016574Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:32.385288238Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:49125"}
{"msg":"region 343d7529-2120-4d07-92fe-d221c2e03516 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:32.405000354Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:32.411344658Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:32.415514589Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"uri":"/crucible/0/regions","method":"POST","req_id":"c52a406b-a85d-43aa-8d29-e549a0df63c6","remote_addr":"[fd00:1122:3344:101::a]:49125","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":16239,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_3902ae36-9ef8-49e4-a803-969ef89cd471/crucible/regions/343d7529-2120-4d07-92fe-d221c2e03516","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:32.422797855Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_3902ae36-9ef8-49e4-a803-969ef89cd471/crucible/regions/343d7529-2120-4d07-92fe-d221c2e03516","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:32.422814355Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"343d7529-2120-4d07-92fe-d221c2e03516\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/343d7529-2120-4d07-92fe-d221c2e03516\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:32.48364879Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"region":"343d7529-2120-4d07-92fe-d221c2e03516","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"343d7529-2120-4d07-92fe-d221c2e03516\" \"--data\" \"/data/regions/343d7529-2120-4d07-92fe-d221c2e03516\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:32.485863515Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"region":"343d7529-2120-4d07-92fe-d221c2e03516","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:32.626508855Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"region":"343d7529-2120-4d07-92fe-d221c2e03516","component":"worker"}
{"msg":"region 343d7529-2120-4d07-92fe-d221c2e03516 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:32.626557247Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:32.62665851Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-343d7529-2120-4d07-92fe-d221c2e03516","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:32.627114726Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-343d7529-2120-4d07-92fe-d221c2e03516","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:32.631610238Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:32.64856655Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-343d7529-2120-4d07-92fe-d221c2e03516","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:32.652057138Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/343d7529-2120-4d07-92fe-d221c2e03516","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:32.658670662Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:32.660617037Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:32.66067755Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:32.660736061Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:32.666254828Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:32.671227446Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:23:32.742590596Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"uri":"/crucible/0/regions","method":"POST","req_id":"82724b52-819e-40a5-b72c-19c3e0413e6c","remote_addr":"[fd00:1122:3344:101::a]:49125","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":94,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.226933088Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:44144"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.227042311Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"uri":"/crucible/0/regions/343d7529-2120-4d07-92fe-d221c2e03516","method":"GET","req_id":"d15e03ec-47b1-4064-9119-e95c2cb7d7e2","remote_addr":"[fd00:1122:3344:101::a]:44144","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":61,"response_code":"200"}
{"msg":"path is \"/data/regions/343d7529-2120-4d07-92fe-d221c2e03516\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.232531637Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"datafile"}
{"msg":"dataset is oxp_3902ae36-9ef8-49e4-a803-969ef89cd471/crucible/regions/343d7529-2120-4d07-92fe-d221c2e03516","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.240970412Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"datafile"}
{"msg":"snapshot is oxp_3902ae36-9ef8-49e4-a803-969ef89cd471/crucible/regions/343d7529-2120-4d07-92fe-d221c2e03516@f6b42f01-d246-4a2c-adb5-3e6bf19c80a4","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.249307733Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_3902ae36-9ef8-49e4-a803-969ef89cd471/crucible/regions/343d7529-2120-4d07-92fe-d221c2e03516\", \"f6b42f01-d246-4a2c-adb5-3e6bf19c80a4\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.249338165Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.260560033Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"uri":"/crucible/0/regions/343d7529-2120-4d07-92fe-d221c2e03516/snapshots/f6b42f01-d246-4a2c-adb5-3e6bf19c80a4","method":"GET","req_id":"380c4248-b6c7-42f3-8754-f312746e87fe","remote_addr":"[fd00:1122:3344:101::a]:44144","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":31273,"response_code":"200"}
{"msg":"path is \"/data/regions/343d7529-2120-4d07-92fe-d221c2e03516\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.26341805Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"datafile"}
{"msg":"dataset is oxp_3902ae36-9ef8-49e4-a803-969ef89cd471/crucible/regions/343d7529-2120-4d07-92fe-d221c2e03516","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.271234304Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"datafile"}
{"msg":"snapshot is oxp_3902ae36-9ef8-49e4-a803-969ef89cd471/crucible/regions/343d7529-2120-4d07-92fe-d221c2e03516@f6b42f01-d246-4a2c-adb5-3e6bf19c80a4","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.279633218Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_3902ae36-9ef8-49e4-a803-969ef89cd471/crucible/regions/343d7529-2120-4d07-92fe-d221c2e03516\", \"f6b42f01-d246-4a2c-adb5-3e6bf19c80a4\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.279665159Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 343d7529-2120-4d07-92fe-d221c2e03516-f6b42f01-d246-4a2c-adb5-3e6bf19c80a4 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.288629972Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"datafile"}
{"msg":"applying SMF actions for region 343d7529-2120-4d07-92fe-d221c2e03516 running snapshot f6b42f01-d246-4a2c-adb5-3e6bf19c80a4 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.2894621Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.289514191Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"uri":"/crucible/0/regions/343d7529-2120-4d07-92fe-d221c2e03516/snapshots/f6b42f01-d246-4a2c-adb5-3e6bf19c80a4/run","method":"POST","req_id":"5d3fa8fd-7979-4b18-8fb8-bfc6a474afcf","remote_addr":"[fd00:1122:3344:101::a]:44144","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":26620,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/343d7529-2120-4d07-92fe-d221c2e03516","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.298014338Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.298110032Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.298190304Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.298198965Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.298239426Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-343d7529-2120-4d07-92fe-d221c2e03516","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.298288338Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/343d7529-2120-4d07-92fe-d221c2e03516","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.298381201Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.298443133Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.298489134Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.298536936Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.30931609Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-343d7529-2120-4d07-92fe-d221c2e03516-f6b42f01-d246-4a2c-adb5-3e6bf19c80a4","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.309771256Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-343d7529-2120-4d07-92fe-d221c2e03516-f6b42f01-d246-4a2c-adb5-3e6bf19c80a4","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.316243944Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-343d7529-2120-4d07-92fe-d221c2e03516-f6b42f01-d246-4a2c-adb5-3e6bf19c80a4","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.339145678Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/343d7529-2120-4d07-92fe-d221c2e03516/.zfs/snapshot/f6b42f01-d246-4a2c-adb5-3e6bf19c80a4","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.339435597Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.33951585Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.339567682Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.339616804Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.339689756Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.348148242Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.355371626Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"region 343d7529-2120-4d07-92fe-d221c2e03516 running snapshot f6b42f01-d246-4a2c-adb5-3e6bf19c80a4 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.355398287Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"datafile"}
{"msg":"path is \"/data/regions/343d7529-2120-4d07-92fe-d221c2e03516\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.466638354Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"datafile"}
{"msg":"dataset is oxp_3902ae36-9ef8-49e4-a803-969ef89cd471/crucible/regions/343d7529-2120-4d07-92fe-d221c2e03516","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.47539522Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"datafile"}
{"msg":"snapshot is oxp_3902ae36-9ef8-49e4-a803-969ef89cd471/crucible/regions/343d7529-2120-4d07-92fe-d221c2e03516@f6b42f01-d246-4a2c-adb5-3e6bf19c80a4","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.483245765Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_3902ae36-9ef8-49e4-a803-969ef89cd471/crucible/regions/343d7529-2120-4d07-92fe-d221c2e03516\", \"f6b42f01-d246-4a2c-adb5-3e6bf19c80a4\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.483271926Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:43.49140053Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"uri":"/crucible/0/regions/343d7529-2120-4d07-92fe-d221c2e03516/snapshots/f6b42f01-d246-4a2c-adb5-3e6bf19c80a4/run","method":"POST","req_id":"44d7965e-0852-4e6b-a21e-0b1ec10821eb","remote_addr":"[fd00:1122:3344:101::a]:44144","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":24799,"response_code":"200"}
{"msg":"region 8475a7ca-5b0b-4a24-b2c7-39d5b3401c93 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.347477457Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.347638382Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"uri":"/crucible/0/regions","method":"POST","req_id":"d6a166fc-f3c8-41f1-a878-48ce654ea019","remote_addr":"[fd00:1122:3344:101::a]:44144","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":224,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.347664723Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_3902ae36-9ef8-49e4-a803-969ef89cd471/crucible/regions/8475a7ca-5b0b-4a24-b2c7-39d5b3401c93","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.357304119Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_3902ae36-9ef8-49e4-a803-969ef89cd471/crucible/regions/8475a7ca-5b0b-4a24-b2c7-39d5b3401c93","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.3573239Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"8475a7ca-5b0b-4a24-b2c7-39d5b3401c93\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/8475a7ca-5b0b-4a24-b2c7-39d5b3401c93\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.460218515Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"region":"8475a7ca-5b0b-4a24-b2c7-39d5b3401c93","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"8475a7ca-5b0b-4a24-b2c7-39d5b3401c93\" \"--data\" \"/data/regions/8475a7ca-5b0b-4a24-b2c7-39d5b3401c93\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.460280717Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"region":"8475a7ca-5b0b-4a24-b2c7-39d5b3401c93","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.613759141Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"region":"8475a7ca-5b0b-4a24-b2c7-39d5b3401c93","component":"worker"}
{"msg":"region 8475a7ca-5b0b-4a24-b2c7-39d5b3401c93 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.613816803Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.613910516Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/343d7529-2120-4d07-92fe-d221c2e03516","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.614542257Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.61461513Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.614695802Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.614702803Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.614743504Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-343d7529-2120-4d07-92fe-d221c2e03516","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.614803346Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/343d7529-2120-4d07-92fe-d221c2e03516","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.615299283Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.615377085Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.615427367Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.61549364Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.623294443Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-8475a7ca-5b0b-4a24-b2c7-39d5b3401c93","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.623810121Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-8475a7ca-5b0b-4a24-b2c7-39d5b3401c93","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.632724642Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.653067049Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-8475a7ca-5b0b-4a24-b2c7-39d5b3401c93","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.670379543Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8475a7ca-5b0b-4a24-b2c7-39d5b3401c93","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.670660073Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.670721765Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.670763436Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.670804698Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.683278739Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.698015987Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.705464548Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"uri":"/crucible/0/regions","method":"POST","req_id":"afcf2922-a3eb-42f9-94b8-fee0d6ce57f7","remote_addr":"[fd00:1122:3344:101::a]:44144","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":71,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.970855992Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:37561"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:28:46.970965866Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"uri":"/crucible/0/regions/8475a7ca-5b0b-4a24-b2c7-39d5b3401c93","method":"GET","req_id":"e9a27b7b-bdb0-42a7-bcc2-82ef9c708be1","remote_addr":"[fd00:1122:3344:101::c]:37561","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":42,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:29:27.173025259Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"uri":"/crucible/0/regions/8475a7ca-5b0b-4a24-b2c7-39d5b3401c93","method":"GET","req_id":"526b78c8-6e5a-4f38-8050-36e4cdcb5276","remote_addr":"[fd00:1122:3344:101::a]:44144","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":517,"response_code":"200"}
{"msg":"path is \"/data/regions/8475a7ca-5b0b-4a24-b2c7-39d5b3401c93\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:29:27.175253714Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"datafile"}
{"msg":"dataset is oxp_3902ae36-9ef8-49e4-a803-969ef89cd471/crucible/regions/8475a7ca-5b0b-4a24-b2c7-39d5b3401c93","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:29:27.184947632Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"datafile"}
{"msg":"region 8475a7ca-5b0b-4a24-b2c7-39d5b3401c93 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:29:27.193507301Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:29:27.193709068Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:29:27.19378427Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"uri":"/crucible/0/regions/8475a7ca-5b0b-4a24-b2c7-39d5b3401c93","method":"DELETE","req_id":"c68b8d8c-961b-4c99-a6d8-a7f44e26a511","remote_addr":"[fd00:1122:3344:101::a]:44144","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":18554,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-8475a7ca-5b0b-4a24-b2c7-39d5b3401c93 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:29:27.195140776Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:29:27.196018016Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"uri":"/crucible/0/regions/8475a7ca-5b0b-4a24-b2c7-39d5b3401c93","method":"GET","req_id":"08db4534-8814-454a-ab54-5841f2dabfbb","remote_addr":"[fd00:1122:3344:101::a]:44144","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":20,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/343d7529-2120-4d07-92fe-d221c2e03516","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:29:27.200054802Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:29:27.200154855Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:29:27.200240508Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:29:27.200248029Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:29:27.20028978Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-343d7529-2120-4d07-92fe-d221c2e03516","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:29:27.200335052Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/343d7529-2120-4d07-92fe-d221c2e03516","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:29:27.200657232Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:29:27.200732645Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:29:27.200781076Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:29:27.200842929Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:29:27.204353497Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:29:27.205690762Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_3902ae36-9ef8-49e4-a803-969ef89cd471/crucible/regions/8475a7ca-5b0b-4a24-b2c7-39d5b3401c93\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:29:27.215238895Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"region":"8475a7ca-5b0b-4a24-b2c7-39d5b3401c93","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:29:27.25978572Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"uri":"/crucible/0/regions/8475a7ca-5b0b-4a24-b2c7-39d5b3401c93","method":"GET","req_id":"71156984-efe0-42b9-9781-db3aef19d9e8","remote_addr":"[fd00:1122:3344:101::a]:44144","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":73,"response_code":"200"}
{"msg":"region 8475a7ca-5b0b-4a24-b2c7-39d5b3401c93 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:29:27.266667592Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-23T04:29:27.325355074Z","hostname":"oxz_crucible_c904874f-485d-4b31-896b-c23c364fb9d6","pid":21160,"uri":"/crucible/0/regions/8475a7ca-5b0b-4a24-b2c7-39d5b3401c93","method":"GET","req_id":"282e3fa3-6fe8-4ae7-ba3e-13e42ec424fb","remote_addr":"[fd00:1122:3344:101::a]:44144","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":24,"response_code":"200"}