[ May  8 03:00:08 Enabled. ]
[ May  8 03:00:08 Rereading configuration. ]
[ May  8 03:00:09 Rereading configuration. ]
[ May  8 03:00:12 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_b59ab028-f607-4bec-945f-ca74fd055dd8/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:00:12.413820272Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976}
{"msg":"dataset: \"oxp_b59ab028-f607-4bec-945f-ca74fd055dd8/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:00:12.414809762Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:00:12.414822153Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:00:12.414828493Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:00:12.428174199Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:00:12.49802471Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.748665792Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:64226"}
{"msg":"region 862c51f1-a711-4064-9a89-5839a57f669f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.780642433Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.781410837Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.781435007Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"uri":"/crucible/0/regions","method":"POST","req_id":"ee43416e-c786-42e9-94be-6a981b31f962","remote_addr":"[fd00:1122:3344:101::b]:64226","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":11599,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_b59ab028-f607-4bec-945f-ca74fd055dd8/crucible/regions/862c51f1-a711-4064-9a89-5839a57f669f","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.787905334Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_b59ab028-f607-4bec-945f-ca74fd055dd8/crucible/regions/862c51f1-a711-4064-9a89-5839a57f669f","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.787918814Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"862c51f1-a711-4064-9a89-5839a57f669f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/862c51f1-a711-4064-9a89-5839a57f669f\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.82894243Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"region":"862c51f1-a711-4064-9a89-5839a57f669f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"862c51f1-a711-4064-9a89-5839a57f669f\" \"--data\" \"/data/regions/862c51f1-a711-4064-9a89-5839a57f669f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.830865889Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"region":"862c51f1-a711-4064-9a89-5839a57f669f","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.948716839Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"region":"862c51f1-a711-4064-9a89-5839a57f669f","component":"worker"}
{"msg":"region 862c51f1-a711-4064-9a89-5839a57f669f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.948764231Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.948886354Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-862c51f1-a711-4064-9a89-5839a57f669f","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.949286887Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-862c51f1-a711-4064-9a89-5839a57f669f","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.954140944Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.964982254Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-862c51f1-a711-4064-9a89-5839a57f669f","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.96619027Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/862c51f1-a711-4064-9a89-5839a57f669f","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.966990495Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.967050296Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.967099928Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.967137349Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.969288374Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.971654796Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:18.046615524Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"uri":"/crucible/0/regions","method":"POST","req_id":"a9b09274-949a-4cf1-a4b8-142e3de1f1da","remote_addr":"[fd00:1122:3344:101::b]:64226","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":63,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.200458226Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:43859"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.200729905Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"uri":"/crucible/0/regions/862c51f1-a711-4064-9a89-5839a57f669f","method":"GET","req_id":"2de4e38f-1c46-4f07-81b4-326a9bf1933b","remote_addr":"[fd00:1122:3344:101::b]:43859","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":58,"response_code":200}
{"msg":"path is \"/data/regions/862c51f1-a711-4064-9a89-5839a57f669f\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.204008145Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"datafile"}
{"msg":"dataset is oxp_b59ab028-f607-4bec-945f-ca74fd055dd8/crucible/regions/862c51f1-a711-4064-9a89-5839a57f669f","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.210286735Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"datafile"}
{"msg":"snapshot is oxp_b59ab028-f607-4bec-945f-ca74fd055dd8/crucible/regions/862c51f1-a711-4064-9a89-5839a57f669f@defa2aee-9ac4-419c-86ae-2ad870019525","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.216777912Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_b59ab028-f607-4bec-945f-ca74fd055dd8/crucible/regions/862c51f1-a711-4064-9a89-5839a57f669f\", \"defa2aee-9ac4-419c-86ae-2ad870019525\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.216799853Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.225177497Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"uri":"/crucible/0/regions/862c51f1-a711-4064-9a89-5839a57f669f/snapshots/defa2aee-9ac4-419c-86ae-2ad870019525","method":"GET","req_id":"16ec3388-e9b9-40a2-8663-b95ad114bd5e","remote_addr":"[fd00:1122:3344:101::b]:43859","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":21189,"response_code":200}
{"msg":"path is \"/data/regions/862c51f1-a711-4064-9a89-5839a57f669f\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.229247921Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"datafile"}
{"msg":"dataset is oxp_b59ab028-f607-4bec-945f-ca74fd055dd8/crucible/regions/862c51f1-a711-4064-9a89-5839a57f669f","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.235304825Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"datafile"}
{"msg":"snapshot is oxp_b59ab028-f607-4bec-945f-ca74fd055dd8/crucible/regions/862c51f1-a711-4064-9a89-5839a57f669f@defa2aee-9ac4-419c-86ae-2ad870019525","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.240550315Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_b59ab028-f607-4bec-945f-ca74fd055dd8/crucible/regions/862c51f1-a711-4064-9a89-5839a57f669f\", \"defa2aee-9ac4-419c-86ae-2ad870019525\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.240585316Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 862c51f1-a711-4064-9a89-5839a57f669f-defa2aee-9ac4-419c-86ae-2ad870019525 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.247283429Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"datafile"}
{"msg":"applying SMF actions for region 862c51f1-a711-4064-9a89-5839a57f669f running snapshot defa2aee-9ac4-419c-86ae-2ad870019525 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.247995051Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.248028072Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"uri":"/crucible/0/regions/862c51f1-a711-4064-9a89-5839a57f669f/snapshots/defa2aee-9ac4-419c-86ae-2ad870019525/run","method":"POST","req_id":"655ec14b-d7af-4f14-a31e-507d4986f397","remote_addr":"[fd00:1122:3344:101::b]:43859","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":18805,"response_code":200}
{"msg":"creating missing snapshot instance snapshot-862c51f1-a711-4064-9a89-5839a57f669f-defa2aee-9ac4-419c-86ae-2ad870019525","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.263828122Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-862c51f1-a711-4064-9a89-5839a57f669f-defa2aee-9ac4-419c-86ae-2ad870019525","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.266293487Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-862c51f1-a711-4064-9a89-5839a57f669f-defa2aee-9ac4-419c-86ae-2ad870019525","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.273738103Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/862c51f1-a711-4064-9a89-5839a57f669f/.zfs/snapshot/defa2aee-9ac4-419c-86ae-2ad870019525","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.274026402Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.274068833Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.274108384Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.274144735Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.274189607Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.276360072Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.279328313Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"worker"}
{"msg":"region 862c51f1-a711-4064-9a89-5839a57f669f running snapshot defa2aee-9ac4-419c-86ae-2ad870019525 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.279343903Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"datafile"}
{"msg":"path is \"/data/regions/862c51f1-a711-4064-9a89-5839a57f669f\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.401481734Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"datafile"}
{"msg":"dataset is oxp_b59ab028-f607-4bec-945f-ca74fd055dd8/crucible/regions/862c51f1-a711-4064-9a89-5839a57f669f","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.408620591Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"datafile"}
{"msg":"snapshot is oxp_b59ab028-f607-4bec-945f-ca74fd055dd8/crucible/regions/862c51f1-a711-4064-9a89-5839a57f669f@defa2aee-9ac4-419c-86ae-2ad870019525","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.414683265Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_b59ab028-f607-4bec-945f-ca74fd055dd8/crucible/regions/862c51f1-a711-4064-9a89-5839a57f669f\", \"defa2aee-9ac4-419c-86ae-2ad870019525\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.414724396Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.42109992Z","hostname":"oxz_crucible_67c71398-1053-41b6-b69e-894ba1479170","pid":26976,"uri":"/crucible/0/regions/862c51f1-a711-4064-9a89-5839a57f669f/snapshots/defa2aee-9ac4-419c-86ae-2ad870019525/run","method":"POST","req_id":"ca8d4830-cae1-41ad-b1d9-cb1725b939ff","remote_addr":"[fd00:1122:3344:101::b]:43859","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":19669,"response_code":200}