[ Mar 25 22:49:25 Enabled. ]
[ Mar 25 22:49:25 Rereading configuration. ]
[ Mar 25 22:49:28 Rereading configuration. ]
[ Mar 25 22:49:33 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_f62b90ae-a5ff-44f8-95be-c523217ebda4/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_f62b90ae-a5ff-44f8-95be-c523217ebda4/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:33.724576075Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:33.728005072Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:33.728014243Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:33.748224198Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:33.838312699Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:52.405380405Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:33774"}
{"msg":"region 21a1331d-4287-4eaa-b22b-046a3b309328 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:52.425616591Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:52.428375033Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"uri":"/crucible/0/regions","method":"POST","req_id":"5c144f6a-e204-4722-8b92-7e4546b55467","remote_addr":"[fd00:1122:3344:101::c]:33774","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":7784,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:52.429039008Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_f62b90ae-a5ff-44f8-95be-c523217ebda4/crucible/regions/21a1331d-4287-4eaa-b22b-046a3b309328","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:52.439042464Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_f62b90ae-a5ff-44f8-95be-c523217ebda4/crucible/regions/21a1331d-4287-4eaa-b22b-046a3b309328","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:52.439124166Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"21a1331d-4287-4eaa-b22b-046a3b309328\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/21a1331d-4287-4eaa-b22b-046a3b309328\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:52.547918928Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"region":"21a1331d-4287-4eaa-b22b-046a3b309328","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"21a1331d-4287-4eaa-b22b-046a3b309328\" \"--data\" \"/data/regions/21a1331d-4287-4eaa-b22b-046a3b309328\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:52.549942714Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"region":"21a1331d-4287-4eaa-b22b-046a3b309328","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:52.735957197Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"uri":"/crucible/0/regions","method":"POST","req_id":"a8d883cb-6221-4057-a656-99ed858c6f83","remote_addr":"[fd00:1122:3344:101::c]:33774","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:52.736467998Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"region":"21a1331d-4287-4eaa-b22b-046a3b309328","component":"worker"}
{"msg":"region 21a1331d-4287-4eaa-b22b-046a3b309328 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:52.73652635Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:52.736762395Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-21a1331d-4287-4eaa-b22b-046a3b309328","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:52.737239276Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-21a1331d-4287-4eaa-b22b-046a3b309328","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:52.7445387Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:52.760787717Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-21a1331d-4287-4eaa-b22b-046a3b309328","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:52.765567814Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/21a1331d-4287-4eaa-b22b-046a3b309328","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:52.766324062Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:52.766401713Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:52.766443324Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:52.766488065Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:52.776230145Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:52.781451283Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:49:53.414488652Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"uri":"/crucible/0/regions","method":"POST","req_id":"aec4f82a-bc9a-407f-b20f-e4878de429dc","remote_addr":"[fd00:1122:3344:101::c]:33774","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":54,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.130883994Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:45481"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.133948773Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"uri":"/crucible/0/regions/21a1331d-4287-4eaa-b22b-046a3b309328","method":"GET","req_id":"9c1bd491-bdd1-40f7-b60d-48ae3903d89b","remote_addr":"[fd00:1122:3344:101::c]:45481","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":3000,"response_code":"200"}
{"msg":"path is \"/data/regions/21a1331d-4287-4eaa-b22b-046a3b309328\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.139924597Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"datafile"}
{"msg":"dataset is oxp_f62b90ae-a5ff-44f8-95be-c523217ebda4/crucible/regions/21a1331d-4287-4eaa-b22b-046a3b309328","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.152095632Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"datafile"}
{"msg":"snapshot is oxp_f62b90ae-a5ff-44f8-95be-c523217ebda4/crucible/regions/21a1331d-4287-4eaa-b22b-046a3b309328@2622e9be-a829-465b-8f08-acde5d239349","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.161546795Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_f62b90ae-a5ff-44f8-95be-c523217ebda4/crucible/regions/21a1331d-4287-4eaa-b22b-046a3b309328\", \"2622e9be-a829-465b-8f08-acde5d239349\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.161583316Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.1724212Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"uri":"/crucible/0/regions/21a1331d-4287-4eaa-b22b-046a3b309328/snapshots/2622e9be-a829-465b-8f08-acde5d239349","method":"GET","req_id":"40a21fa4-c943-4900-a16a-ba062fb20ff6","remote_addr":"[fd00:1122:3344:101::c]:45481","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":35003,"response_code":"200"}
{"msg":"path is \"/data/regions/21a1331d-4287-4eaa-b22b-046a3b309328\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.175752145Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"datafile"}
{"msg":"dataset is oxp_f62b90ae-a5ff-44f8-95be-c523217ebda4/crucible/regions/21a1331d-4287-4eaa-b22b-046a3b309328","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.183822287Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"datafile"}
{"msg":"snapshot is oxp_f62b90ae-a5ff-44f8-95be-c523217ebda4/crucible/regions/21a1331d-4287-4eaa-b22b-046a3b309328@2622e9be-a829-465b-8f08-acde5d239349","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.191865718Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_f62b90ae-a5ff-44f8-95be-c523217ebda4/crucible/regions/21a1331d-4287-4eaa-b22b-046a3b309328\", \"2622e9be-a829-465b-8f08-acde5d239349\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.191897199Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 21a1331d-4287-4eaa-b22b-046a3b309328-2622e9be-a829-465b-8f08-acde5d239349 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.200780909Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"datafile"}
{"msg":"applying SMF actions for region 21a1331d-4287-4eaa-b22b-046a3b309328 running snapshot 2622e9be-a829-465b-8f08-acde5d239349 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.201336932Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.201357732Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"uri":"/crucible/0/regions/21a1331d-4287-4eaa-b22b-046a3b309328/snapshots/2622e9be-a829-465b-8f08-acde5d239349/run","method":"POST","req_id":"53f71b49-9d83-4d19-b2fe-1439c2190672","remote_addr":"[fd00:1122:3344:101::c]:45481","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":25639,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-21a1331d-4287-4eaa-b22b-046a3b309328-2622e9be-a829-465b-8f08-acde5d239349","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.209352882Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-21a1331d-4287-4eaa-b22b-046a3b309328-2622e9be-a829-465b-8f08-acde5d239349","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.21368276Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-21a1331d-4287-4eaa-b22b-046a3b309328-2622e9be-a829-465b-8f08-acde5d239349","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.225931956Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/21a1331d-4287-4eaa-b22b-046a3b309328/.zfs/snapshot/2622e9be-a829-465b-8f08-acde5d239349","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.226239403Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.226295615Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.226343885Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.226384106Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.226433098Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.230149301Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.234649933Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"worker"}
{"msg":"region 21a1331d-4287-4eaa-b22b-046a3b309328 running snapshot 2622e9be-a829-465b-8f08-acde5d239349 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.234666573Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"datafile"}
{"msg":"path is \"/data/regions/21a1331d-4287-4eaa-b22b-046a3b309328\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.504537436Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"datafile"}
{"msg":"dataset is oxp_f62b90ae-a5ff-44f8-95be-c523217ebda4/crucible/regions/21a1331d-4287-4eaa-b22b-046a3b309328","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.513279013Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"datafile"}
{"msg":"snapshot is oxp_f62b90ae-a5ff-44f8-95be-c523217ebda4/crucible/regions/21a1331d-4287-4eaa-b22b-046a3b309328@2622e9be-a829-465b-8f08-acde5d239349","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.521723474Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_f62b90ae-a5ff-44f8-95be-c523217ebda4/crucible/regions/21a1331d-4287-4eaa-b22b-046a3b309328\", \"2622e9be-a829-465b-8f08-acde5d239349\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.521767225Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:59:55.531930094Z","hostname":"oxz_crucible_d7cf2f10-5272-45b6-8af6-6502b1b5f879","pid":25528,"uri":"/crucible/0/regions/21a1331d-4287-4eaa-b22b-046a3b309328/snapshots/2622e9be-a829-465b-8f08-acde5d239349/run","method":"POST","req_id":"ec62248d-9617-4f12-8fdd-2d397e569b0c","remote_addr":"[fd00:1122:3344:101::c]:45481","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":27420,"response_code":"200"}