[ Oct 28 19:00:57 Enabled. ]
[ Oct 28 19:00:57 Rereading configuration. ]
[ Oct 28 19:00:59 Rereading configuration. ]
[ Oct 28 19:01:02 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_a166e4c1-e87f-45a1-b904-40fe3eb95e83/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: f85cf0dc3631e2f0c13384787efb0efc1b642162\nCommit timestamp: 2026-10-10T00:38:51.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:02.956809147Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925}
{"msg":"dataset: \"oxp_a166e4c1-e87f-45a1-b904-40fe3eb95e83/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:02.957014723Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:02.957029904Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:02.957039564Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:02.975850609Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:03.046332316Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:30.511622576Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:39656"}
{"msg":"region faf05342-d17e-40c5-bf27-6a41fdbc516c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:30.541042057Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:30.545422919Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:30.549867543Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"uri":"/crucible/0/regions","method":"POST","req_id":"6a018758-a32f-4712-9be8-58e4abc3c5cc","remote_addr":"[fd00:1122:3344:101::c]:39656","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":15564,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_a166e4c1-e87f-45a1-b904-40fe3eb95e83/crucible/regions/faf05342-d17e-40c5-bf27-6a41fdbc516c","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:30.552390042Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_a166e4c1-e87f-45a1-b904-40fe3eb95e83/crucible/regions/faf05342-d17e-40c5-bf27-6a41fdbc516c","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:30.552421777Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"faf05342-d17e-40c5-bf27-6a41fdbc516c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/faf05342-d17e-40c5-bf27-6a41fdbc516c\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:30.624881797Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"region":"faf05342-d17e-40c5-bf27-6a41fdbc516c","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"faf05342-d17e-40c5-bf27-6a41fdbc516c\" \"--data\" \"/data/regions/faf05342-d17e-40c5-bf27-6a41fdbc516c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:30.624952169Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"region":"faf05342-d17e-40c5-bf27-6a41fdbc516c","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:30.755954465Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"region":"faf05342-d17e-40c5-bf27-6a41fdbc516c","component":"worker"}
{"msg":"region faf05342-d17e-40c5-bf27-6a41fdbc516c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:30.758487436Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:30.758774424Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-faf05342-d17e-40c5-bf27-6a41fdbc516c","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:30.759098062Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-faf05342-d17e-40c5-bf27-6a41fdbc516c","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:30.762972131Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:30.769723659Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-faf05342-d17e-40c5-bf27-6a41fdbc516c","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:30.771189616Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/faf05342-d17e-40c5-bf27-6a41fdbc516c","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:30.771880957Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:30.771930983Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:30.772042654Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:30.772072985Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:30.774642356Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:30.777138206Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:01:30.844199648Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"uri":"/crucible/0/regions","method":"POST","req_id":"8d1e49ef-ac36-4b0f-9118-0dc36724f159","remote_addr":"[fd00:1122:3344:101::c]:39656","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":80,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.464823305Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:42907"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.465147964Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"uri":"/crucible/0/regions/faf05342-d17e-40c5-bf27-6a41fdbc516c","method":"GET","req_id":"0e9b3ee8-479e-437b-99f7-674c31aed8b9","remote_addr":"[fd00:1122:3344:101::c]:42907","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":47,"response_code":200}
{"msg":"path is \"/data/regions/faf05342-d17e-40c5-bf27-6a41fdbc516c\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.470944405Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"datafile"}
{"msg":"dataset is oxp_a166e4c1-e87f-45a1-b904-40fe3eb95e83/crucible/regions/faf05342-d17e-40c5-bf27-6a41fdbc516c","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.476529201Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"datafile"}
{"msg":"snapshot is oxp_a166e4c1-e87f-45a1-b904-40fe3eb95e83/crucible/regions/faf05342-d17e-40c5-bf27-6a41fdbc516c@0cd37316-8e9c-4dfb-a2a1-a0db5abb2761","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.481576762Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_a166e4c1-e87f-45a1-b904-40fe3eb95e83/crucible/regions/faf05342-d17e-40c5-bf27-6a41fdbc516c\", \"0cd37316-8e9c-4dfb-a2a1-a0db5abb2761\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.481591433Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.49260961Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"uri":"/crucible/0/regions/faf05342-d17e-40c5-bf27-6a41fdbc516c/snapshots/0cd37316-8e9c-4dfb-a2a1-a0db5abb2761","method":"GET","req_id":"598ee948-9188-4f4d-9298-73dbfcbdda72","remote_addr":"[fd00:1122:3344:101::c]:42907","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":24147,"response_code":200}
{"msg":"path is \"/data/regions/faf05342-d17e-40c5-bf27-6a41fdbc516c\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.496085847Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"datafile"}
{"msg":"dataset is oxp_a166e4c1-e87f-45a1-b904-40fe3eb95e83/crucible/regions/faf05342-d17e-40c5-bf27-6a41fdbc516c","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.501742955Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"datafile"}
{"msg":"snapshot is oxp_a166e4c1-e87f-45a1-b904-40fe3eb95e83/crucible/regions/faf05342-d17e-40c5-bf27-6a41fdbc516c@0cd37316-8e9c-4dfb-a2a1-a0db5abb2761","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.506975931Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_a166e4c1-e87f-45a1-b904-40fe3eb95e83/crucible/regions/faf05342-d17e-40c5-bf27-6a41fdbc516c\", \"0cd37316-8e9c-4dfb-a2a1-a0db5abb2761\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.506991452Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot faf05342-d17e-40c5-bf27-6a41fdbc516c-0cd37316-8e9c-4dfb-a2a1-a0db5abb2761 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.513716559Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.514321976Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"uri":"/crucible/0/regions/faf05342-d17e-40c5-bf27-6a41fdbc516c/snapshots/0cd37316-8e9c-4dfb-a2a1-a0db5abb2761/run","method":"POST","req_id":"a0706e45-7011-4c00-b08b-ab039e297469","remote_addr":"[fd00:1122:3344:101::c]:42907","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":18245,"response_code":200}
{"msg":"applying SMF actions for region faf05342-d17e-40c5-bf27-6a41fdbc516c running snapshot 0cd37316-8e9c-4dfb-a2a1-a0db5abb2761 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.514353887Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-faf05342-d17e-40c5-bf27-6a41fdbc516c-0cd37316-8e9c-4dfb-a2a1-a0db5abb2761","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.526461255Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-faf05342-d17e-40c5-bf27-6a41fdbc516c-0cd37316-8e9c-4dfb-a2a1-a0db5abb2761","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.52807371Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-faf05342-d17e-40c5-bf27-6a41fdbc516c-0cd37316-8e9c-4dfb-a2a1-a0db5abb2761","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.538379078Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/faf05342-d17e-40c5-bf27-6a41fdbc516c/.zfs/snapshot/0cd37316-8e9c-4dfb-a2a1-a0db5abb2761","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.538635715Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.538686276Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.538725608Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.538765518Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.53882012Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.54098076Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.544035116Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"worker"}
{"msg":"region faf05342-d17e-40c5-bf27-6a41fdbc516c running snapshot 0cd37316-8e9c-4dfb-a2a1-a0db5abb2761 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.544048616Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"datafile"}
{"msg":"path is \"/data/regions/faf05342-d17e-40c5-bf27-6a41fdbc516c\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.8293721Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"datafile"}
{"msg":"dataset is oxp_a166e4c1-e87f-45a1-b904-40fe3eb95e83/crucible/regions/faf05342-d17e-40c5-bf27-6a41fdbc516c","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.835691366Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"datafile"}
{"msg":"snapshot is oxp_a166e4c1-e87f-45a1-b904-40fe3eb95e83/crucible/regions/faf05342-d17e-40c5-bf27-6a41fdbc516c@0cd37316-8e9c-4dfb-a2a1-a0db5abb2761","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.842583009Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_a166e4c1-e87f-45a1-b904-40fe3eb95e83/crucible/regions/faf05342-d17e-40c5-bf27-6a41fdbc516c\", \"0cd37316-8e9c-4dfb-a2a1-a0db5abb2761\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.84262784Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-28T19:12:25.849182693Z","hostname":"oxz_crucible_edacc772-2a60-41ef-9035-aacb218d7ee9","pid":24925,"uri":"/crucible/0/regions/faf05342-d17e-40c5-bf27-6a41fdbc516c/snapshots/0cd37316-8e9c-4dfb-a2a1-a0db5abb2761/run","method":"POST","req_id":"3c2f1f05-ba62-4711-b884-1857f229c5f4","remote_addr":"[fd00:1122:3344:101::c]:42907","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":19869,"response_code":200}