[ Jul 18 19:14:56 Enabled. ]
[ Jul 18 19:14:56 Rereading configuration. ]
[ Jul 18 19:15:00 Rereading configuration. ]
[ Jul 18 19:15:05 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_d434fc66-9378-47a1-bac3-8eaeb65e5efa/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_d434fc66-9378-47a1-bac3-8eaeb65e5efa/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:05.157446014Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:05.160679807Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:05.160692387Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:05.177318575Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:05.35732208Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.001872314Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:35775"}
{"msg":"region f19df003-f2d8-4868-87c4-037da4de73f8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.045531815Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.054122137Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"uri":"/crucible/0/regions","method":"POST","req_id":"1f6f4cdf-d552-4495-af69-c3b6d885be39","remote_addr":"[fd00:1122:3344:101::a]:35775","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":27953,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.056622215Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_d434fc66-9378-47a1-bac3-8eaeb65e5efa/crucible/regions/f19df003-f2d8-4868-87c4-037da4de73f8","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.064973174Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_d434fc66-9378-47a1-bac3-8eaeb65e5efa/crucible/regions/f19df003-f2d8-4868-87c4-037da4de73f8","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.065030514Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f19df003-f2d8-4868-87c4-037da4de73f8\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f19df003-f2d8-4868-87c4-037da4de73f8\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.143371869Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"region":"f19df003-f2d8-4868-87c4-037da4de73f8","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f19df003-f2d8-4868-87c4-037da4de73f8\" \"--data\" \"/data/regions/f19df003-f2d8-4868-87c4-037da4de73f8\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.145299439Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"region":"f19df003-f2d8-4868-87c4-037da4de73f8","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.275800715Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"uri":"/crucible/0/regions","method":"POST","req_id":"39ee1443-3da7-4eb5-8326-d0cb250a82a6","remote_addr":"[fd00:1122:3344:101::a]:35775","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":67,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.463361999Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"region":"f19df003-f2d8-4868-87c4-037da4de73f8","component":"worker"}
{"msg":"region f19df003-f2d8-4868-87c4-037da4de73f8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.46343807Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.463643833Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f19df003-f2d8-4868-87c4-037da4de73f8","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.464020939Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f19df003-f2d8-4868-87c4-037da4de73f8","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.469472543Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.48096256Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f19df003-f2d8-4868-87c4-037da4de73f8","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.483965056Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f19df003-f2d8-4868-87c4-037da4de73f8","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.484676407Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.484746748Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.484786299Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.484825689Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.488514026Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.503303133Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:15:31.957174831Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"uri":"/crucible/0/regions","method":"POST","req_id":"0125c74e-9671-4b9b-adff-461fc8991a96","remote_addr":"[fd00:1122:3344:101::a]:35775","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":50,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:14.751447986Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:32920"}
{"msg":"region c9a388be-2ad2-48f3-9bc9-1d62075d5025 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:14.75173646Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:14.752115966Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"uri":"/crucible/0/regions","method":"POST","req_id":"bbd1f272-85d1-41b8-b18a-a91f1479c2b0","remote_addr":"[fd00:1122:3344:101::a]:32920","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":613,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:14.752133006Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_d434fc66-9378-47a1-bac3-8eaeb65e5efa/crucible/regions/c9a388be-2ad2-48f3-9bc9-1d62075d5025","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:14.774959427Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_d434fc66-9378-47a1-bac3-8eaeb65e5efa/crucible/regions/c9a388be-2ad2-48f3-9bc9-1d62075d5025","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:14.775002748Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"c9a388be-2ad2-48f3-9bc9-1d62075d5025\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c9a388be-2ad2-48f3-9bc9-1d62075d5025\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:14.852476259Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"region":"c9a388be-2ad2-48f3-9bc9-1d62075d5025","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c9a388be-2ad2-48f3-9bc9-1d62075d5025\" \"--data\" \"/data/regions/c9a388be-2ad2-48f3-9bc9-1d62075d5025\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:14.85253472Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"region":"c9a388be-2ad2-48f3-9bc9-1d62075d5025","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:14.967884153Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"region":"c9a388be-2ad2-48f3-9bc9-1d62075d5025","component":"worker"}
{"msg":"region c9a388be-2ad2-48f3-9bc9-1d62075d5025 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:14.967928234Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:14.968015135Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-c9a388be-2ad2-48f3-9bc9-1d62075d5025","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:14.968383751Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c9a388be-2ad2-48f3-9bc9-1d62075d5025","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:14.97940359Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:15.004023029Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c9a388be-2ad2-48f3-9bc9-1d62075d5025","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:15.0086568Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c9a388be-2ad2-48f3-9bc9-1d62075d5025","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:15.009019386Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:15.009099537Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:15.009158048Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:15.009216199Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:15.014637032Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:15.021712351Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:15.083518721Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"uri":"/crucible/0/regions","method":"POST","req_id":"7bd369e6-397d-424f-9b78-efad2262bb7a","remote_addr":"[fd00:1122:3344:101::a]:32920","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":55,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.266027652Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"uri":"/crucible/0/regions/f19df003-f2d8-4868-87c4-037da4de73f8","method":"GET","req_id":"688ff8c8-b543-453a-9ebe-a77cd70715c1","remote_addr":"[fd00:1122:3344:101::a]:32920","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":3051,"response_code":"200"}
{"msg":"path is \"/data/regions/f19df003-f2d8-4868-87c4-037da4de73f8\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.2697916Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"datafile"}
{"msg":"dataset is oxp_d434fc66-9378-47a1-bac3-8eaeb65e5efa/crucible/regions/f19df003-f2d8-4868-87c4-037da4de73f8","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.278752358Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"datafile"}
{"msg":"snapshot is oxp_d434fc66-9378-47a1-bac3-8eaeb65e5efa/crucible/regions/f19df003-f2d8-4868-87c4-037da4de73f8@116a239b-954f-4639-80ac-bb95bc129003","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.288081881Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_d434fc66-9378-47a1-bac3-8eaeb65e5efa/crucible/regions/f19df003-f2d8-4868-87c4-037da4de73f8\", \"116a239b-954f-4639-80ac-bb95bc129003\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.288120642Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.299469616Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"uri":"/crucible/0/regions/f19df003-f2d8-4868-87c4-037da4de73f8/snapshots/116a239b-954f-4639-80ac-bb95bc129003","method":"GET","req_id":"97c43749-bdf1-4abc-adef-9f0754bf7bb6","remote_addr":"[fd00:1122:3344:101::a]:32920","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":29671,"response_code":"200"}
{"msg":"path is \"/data/regions/f19df003-f2d8-4868-87c4-037da4de73f8\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.302817868Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"datafile"}
{"msg":"dataset is oxp_d434fc66-9378-47a1-bac3-8eaeb65e5efa/crucible/regions/f19df003-f2d8-4868-87c4-037da4de73f8","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.312135701Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"datafile"}
{"msg":"snapshot is oxp_d434fc66-9378-47a1-bac3-8eaeb65e5efa/crucible/regions/f19df003-f2d8-4868-87c4-037da4de73f8@116a239b-954f-4639-80ac-bb95bc129003","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.321132849Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_d434fc66-9378-47a1-bac3-8eaeb65e5efa/crucible/regions/f19df003-f2d8-4868-87c4-037da4de73f8\", \"116a239b-954f-4639-80ac-bb95bc129003\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.32116689Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot f19df003-f2d8-4868-87c4-037da4de73f8-116a239b-954f-4639-80ac-bb95bc129003 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.331382947Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"datafile"}
{"msg":"applying SMF actions for region f19df003-f2d8-4868-87c4-037da4de73f8 running snapshot 116a239b-954f-4639-80ac-bb95bc129003 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.3315963Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.33161261Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"uri":"/crucible/0/regions/f19df003-f2d8-4868-87c4-037da4de73f8/snapshots/116a239b-954f-4639-80ac-bb95bc129003/run","method":"POST","req_id":"4f57778a-32ff-4c08-b745-f6cb5af6571f","remote_addr":"[fd00:1122:3344:101::a]:32920","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":28817,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-f19df003-f2d8-4868-87c4-037da4de73f8-116a239b-954f-4639-80ac-bb95bc129003","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.333580091Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-f19df003-f2d8-4868-87c4-037da4de73f8-116a239b-954f-4639-80ac-bb95bc129003","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.342807853Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-f19df003-f2d8-4868-87c4-037da4de73f8-116a239b-954f-4639-80ac-bb95bc129003","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.358281371Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f19df003-f2d8-4868-87c4-037da4de73f8/.zfs/snapshot/116a239b-954f-4639-80ac-bb95bc129003","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.358647796Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.358698077Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.358735487Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.358772988Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.358818239Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.369170208Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.376316078Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"region f19df003-f2d8-4868-87c4-037da4de73f8 running snapshot 116a239b-954f-4639-80ac-bb95bc129003 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.376361518Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"datafile"}
{"msg":"path is \"/data/regions/f19df003-f2d8-4868-87c4-037da4de73f8\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.576632427Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"datafile"}
{"msg":"dataset is oxp_d434fc66-9378-47a1-bac3-8eaeb65e5efa/crucible/regions/f19df003-f2d8-4868-87c4-037da4de73f8","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.585686576Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"datafile"}
{"msg":"snapshot is oxp_d434fc66-9378-47a1-bac3-8eaeb65e5efa/crucible/regions/f19df003-f2d8-4868-87c4-037da4de73f8@116a239b-954f-4639-80ac-bb95bc129003","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.594924668Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_d434fc66-9378-47a1-bac3-8eaeb65e5efa/crucible/regions/f19df003-f2d8-4868-87c4-037da4de73f8\", \"116a239b-954f-4639-80ac-bb95bc129003\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.594965309Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:16.603228346Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"uri":"/crucible/0/regions/f19df003-f2d8-4868-87c4-037da4de73f8/snapshots/116a239b-954f-4639-80ac-bb95bc129003/run","method":"POST","req_id":"2f418daa-777b-4646-81f8-74f594450a32","remote_addr":"[fd00:1122:3344:101::a]:32920","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":26620,"response_code":"200"}
{"msg":"region b188987f-9146-4344-8372-56a2ec1e4352 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:21.320566594Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:21.320748167Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"uri":"/crucible/0/regions","method":"POST","req_id":"6696049c-9f16-4f65-afb8-b6428e21e442","remote_addr":"[fd00:1122:3344:101::a]:32920","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":238,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:21.320778347Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_d434fc66-9378-47a1-bac3-8eaeb65e5efa/crucible/regions/b188987f-9146-4344-8372-56a2ec1e4352","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:21.328926353Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_d434fc66-9378-47a1-bac3-8eaeb65e5efa/crucible/regions/b188987f-9146-4344-8372-56a2ec1e4352","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:21.328957493Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b188987f-9146-4344-8372-56a2ec1e4352\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b188987f-9146-4344-8372-56a2ec1e4352\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:21.384374575Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"region":"b188987f-9146-4344-8372-56a2ec1e4352","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b188987f-9146-4344-8372-56a2ec1e4352\" \"--data\" \"/data/regions/b188987f-9146-4344-8372-56a2ec1e4352\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:21.386319795Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"region":"b188987f-9146-4344-8372-56a2ec1e4352","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:21.505128522Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"region":"b188987f-9146-4344-8372-56a2ec1e4352","component":"worker"}
{"msg":"region b188987f-9146-4344-8372-56a2ec1e4352 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:21.505161032Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:21.505284224Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b188987f-9146-4344-8372-56a2ec1e4352","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:21.505722681Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b188987f-9146-4344-8372-56a2ec1e4352","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:21.509051442Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:21.518118181Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b188987f-9146-4344-8372-56a2ec1e4352","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:21.524220505Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b188987f-9146-4344-8372-56a2ec1e4352","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:21.524493489Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:21.52454774Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:21.524587411Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:21.524628982Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:21.529955924Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:21.542846672Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:21.63899026Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"uri":"/crucible/0/regions","method":"POST","req_id":"eff4f3e3-3626-48d4-9690-9c1088e1c1e6","remote_addr":"[fd00:1122:3344:101::a]:32920","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":87,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:47.433287263Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"uri":"/crucible/0/regions/b188987f-9146-4344-8372-56a2ec1e4352","method":"GET","req_id":"0e1a7920-a92a-4320-bbcc-f442fc3c0204","remote_addr":"[fd00:1122:3344:101::a]:32920","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":50,"response_code":"200"}
{"msg":"path is \"/data/regions/b188987f-9146-4344-8372-56a2ec1e4352\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:47.436200528Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"datafile"}
{"msg":"dataset is oxp_d434fc66-9378-47a1-bac3-8eaeb65e5efa/crucible/regions/b188987f-9146-4344-8372-56a2ec1e4352","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:47.452452138Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"datafile"}
{"msg":"region b188987f-9146-4344-8372-56a2ec1e4352 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:47.46231605Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:47.462546963Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-b188987f-9146-4344-8372-56a2ec1e4352 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:47.463774572Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:47.465746732Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"uri":"/crucible/0/regions/b188987f-9146-4344-8372-56a2ec1e4352","method":"DELETE","req_id":"de58e5fd-4a21-40f3-8535-307c5d133e69","remote_addr":"[fd00:1122:3344:101::a]:32920","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":29568,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:47.468927781Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"uri":"/crucible/0/regions/b188987f-9146-4344-8372-56a2ec1e4352","method":"GET","req_id":"79f5b375-2a3f-450e-8295-873e12c89117","remote_addr":"[fd00:1122:3344:101::a]:32920","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":26,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:47.476357495Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_d434fc66-9378-47a1-bac3-8eaeb65e5efa/crucible/regions/b188987f-9146-4344-8372-56a2ec1e4352\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:47.487201502Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"region":"b188987f-9146-4344-8372-56a2ec1e4352","component":"worker"}
{"msg":"region b188987f-9146-4344-8372-56a2ec1e4352 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:47.537190471Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-18T19:26:47.605099704Z","hostname":"oxz_crucible_043e1bb4-0b83-4e03-befc-d664846db10b","pid":28045,"uri":"/crucible/0/regions/b188987f-9146-4344-8372-56a2ec1e4352","method":"GET","req_id":"8b1f474b-d5a5-4ba3-a0b4-4e9ec57c7bc8","remote_addr":"[fd00:1122:3344:101::a]:32920","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":65,"response_code":"200"}