[ Sep 19 07:31:46 Enabled. ]
[ Sep 19 07:31:46 Rereading configuration. ]
[ Sep 19 07:31:49 Rereading configuration. ]
[ Sep 19 07:31:52 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_6ee68421-c881-4fd9-b9ee-a17cfc057f47/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: ad8a31742adc45e925e63443a5b43c8e30604022\nCommit timestamp: 2026-07-28T19:41:59.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:31:52.111089079Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114}
{"msg":"dataset: \"oxp_6ee68421-c881-4fd9-b9ee-a17cfc057f47/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:31:52.111289134Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:31:52.111307704Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:31:52.111316935Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:31:52.130152521Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:31:52.179924468Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:14.733667965Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:42839"}
{"msg":"region 49849000-8977-4900-9509-8317dd8e3107 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:14.780470503Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:14.785891481Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:14.792894977Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"uri":"/crucible/0/regions","method":"POST","req_id":"cebf2850-8176-47d2-b7ec-bb4d887ecd2e","remote_addr":"[fd00:1122:3344:101::b]:42839","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":23034,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_6ee68421-c881-4fd9-b9ee-a17cfc057f47/crucible/regions/49849000-8977-4900-9509-8317dd8e3107","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:14.794578297Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_6ee68421-c881-4fd9-b9ee-a17cfc057f47/crucible/regions/49849000-8977-4900-9509-8317dd8e3107","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:14.794596607Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"49849000-8977-4900-9509-8317dd8e3107\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/49849000-8977-4900-9509-8317dd8e3107\"","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:14.841881976Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"region":"49849000-8977-4900-9509-8317dd8e3107","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"49849000-8977-4900-9509-8317dd8e3107\" \"--data\" \"/data/regions/49849000-8977-4900-9509-8317dd8e3107\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:14.8445774Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"region":"49849000-8977-4900-9509-8317dd8e3107","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:14.980107737Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"uri":"/crucible/0/regions","method":"POST","req_id":"0cc088bd-1c26-448c-9642-d1a63c9d4c10","remote_addr":"[fd00:1122:3344:101::b]:42839","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":64,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:15.200401761Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"region":"49849000-8977-4900-9509-8317dd8e3107","component":"worker"}
{"msg":"region 49849000-8977-4900-9509-8317dd8e3107 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:15.205927182Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:15.206069305Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-49849000-8977-4900-9509-8317dd8e3107","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:15.21005185Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-49849000-8977-4900-9509-8317dd8e3107","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:15.2189103Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:15.228334632Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-49849000-8977-4900-9509-8317dd8e3107","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:15.229666274Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/49849000-8977-4900-9509-8317dd8e3107","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:15.230391881Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:15.230447673Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:15.230500164Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:15.230532184Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:15.233457354Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:15.236461585Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:15.736789166Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"uri":"/crucible/0/regions","method":"POST","req_id":"21b2ce40-0502-4afb-90ee-28486b77afcc","remote_addr":"[fd00:1122:3344:101::b]:42839","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":177,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:16.157975484Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"uri":"/crucible/0/regions/49849000-8977-4900-9509-8317dd8e3107","method":"GET","req_id":"2c47b4aa-806b-472c-8dcf-1fc274b34f53","remote_addr":"[fd00:1122:3344:101::b]:42839","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":80,"response_code":200}
{"msg":"path is \"/data/regions/49849000-8977-4900-9509-8317dd8e3107\"","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:16.162081631Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"datafile"}
{"msg":"dataset is oxp_6ee68421-c881-4fd9-b9ee-a17cfc057f47/crucible/regions/49849000-8977-4900-9509-8317dd8e3107","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:16.169742473Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"datafile"}
{"msg":"region 49849000-8977-4900-9509-8317dd8e3107 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:16.176498013Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:16.176639046Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:16.176654956Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"uri":"/crucible/0/regions/49849000-8977-4900-9509-8317dd8e3107","method":"DELETE","req_id":"4d8d15e9-c1dc-492f-bb38-ec507ba749a2","remote_addr":"[fd00:1122:3344:101::b]:42839","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":14609,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-49849000-8977-4900-9509-8317dd8e3107 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:16.17765153Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:16.18060701Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:16.180799234Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"uri":"/crucible/0/regions/49849000-8977-4900-9509-8317dd8e3107","method":"GET","req_id":"047b5eb7-ed6a-47fc-bca4-c02e3b7807fa","remote_addr":"[fd00:1122:3344:101::b]:42839","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":53,"response_code":200}
{"msg":"deleting zfs dataset \"oxp_6ee68421-c881-4fd9-b9ee-a17cfc057f47/crucible/regions/49849000-8977-4900-9509-8317dd8e3107\"","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:16.189176853Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"region":"49849000-8977-4900-9509-8317dd8e3107","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:16.244519813Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"uri":"/crucible/0/regions/49849000-8977-4900-9509-8317dd8e3107","method":"GET","req_id":"0968911d-05c5-4102-8525-b5d74fd07a53","remote_addr":"[fd00:1122:3344:101::b]:42839","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":54,"response_code":200}
{"msg":"region 49849000-8977-4900-9509-8317dd8e3107 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:16.24481715Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:16.329026923Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"uri":"/crucible/0/regions/49849000-8977-4900-9509-8317dd8e3107","method":"GET","req_id":"3adf38fc-5c06-485b-b910-5e153598ca4d","remote_addr":"[fd00:1122:3344:101::b]:42839","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":54,"response_code":200}
{"msg":"region 23f3c2d4-c9f8-40af-9506-e644e1b2984d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:19.298709976Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:19.29886496Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:19.298903081Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"uri":"/crucible/0/regions","method":"POST","req_id":"94f2b84f-66ff-40ff-894e-c2f8059b246e","remote_addr":"[fd00:1122:3344:101::b]:42839","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":196,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_6ee68421-c881-4fd9-b9ee-a17cfc057f47/crucible/regions/23f3c2d4-c9f8-40af-9506-e644e1b2984d","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:19.304755239Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_6ee68421-c881-4fd9-b9ee-a17cfc057f47/crucible/regions/23f3c2d4-c9f8-40af-9506-e644e1b2984d","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:19.30479163Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"23f3c2d4-c9f8-40af-9506-e644e1b2984d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/23f3c2d4-c9f8-40af-9506-e644e1b2984d\"","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:19.350099662Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"region":"23f3c2d4-c9f8-40af-9506-e644e1b2984d","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"23f3c2d4-c9f8-40af-9506-e644e1b2984d\" \"--data\" \"/data/regions/23f3c2d4-c9f8-40af-9506-e644e1b2984d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:19.350142934Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"region":"23f3c2d4-c9f8-40af-9506-e644e1b2984d","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:19.398399776Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"region":"23f3c2d4-c9f8-40af-9506-e644e1b2984d","component":"worker"}
{"msg":"region 23f3c2d4-c9f8-40af-9506-e644e1b2984d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:19.398436766Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:19.398527679Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-23f3c2d4-c9f8-40af-9506-e644e1b2984d","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:19.399360648Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-23f3c2d4-c9f8-40af-9506-e644e1b2984d","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:19.4006863Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:19.405031873Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-23f3c2d4-c9f8-40af-9506-e644e1b2984d","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:19.406443846Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/23f3c2d4-c9f8-40af-9506-e644e1b2984d","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:19.406755343Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:19.406817375Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:19.406844565Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:19.406890607Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:19.408758691Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:19.411909525Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:32:19.454313689Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"uri":"/crucible/0/regions","method":"POST","req_id":"f3c582e1-c8d6-4b14-bbf1-28b554144db2","remote_addr":"[fd00:1122:3344:101::b]:42839","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":60,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:29.165010971Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:46751"}
{"msg":"region e826bc23-4a5f-43b8-a25c-7f4483e41699 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:29.165162304Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:29.166129827Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:29.166159248Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"uri":"/crucible/0/regions","method":"POST","req_id":"0bebb031-41bd-4947-930f-07ae111cff8c","remote_addr":"[fd00:1122:3344:101::b]:46751","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":1077,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_6ee68421-c881-4fd9-b9ee-a17cfc057f47/crucible/regions/e826bc23-4a5f-43b8-a25c-7f4483e41699","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:29.17470454Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_6ee68421-c881-4fd9-b9ee-a17cfc057f47/crucible/regions/e826bc23-4a5f-43b8-a25c-7f4483e41699","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:29.17472351Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e826bc23-4a5f-43b8-a25c-7f4483e41699\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e826bc23-4a5f-43b8-a25c-7f4483e41699\"","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:29.219341196Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"region":"e826bc23-4a5f-43b8-a25c-7f4483e41699","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e826bc23-4a5f-43b8-a25c-7f4483e41699\" \"--data\" \"/data/regions/e826bc23-4a5f-43b8-a25c-7f4483e41699\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:29.219380407Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"region":"e826bc23-4a5f-43b8-a25c-7f4483e41699","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:29.265686493Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"region":"e826bc23-4a5f-43b8-a25c-7f4483e41699","component":"worker"}
{"msg":"region e826bc23-4a5f-43b8-a25c-7f4483e41699 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:29.265728044Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:29.265843827Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e826bc23-4a5f-43b8-a25c-7f4483e41699","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:29.275359922Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e826bc23-4a5f-43b8-a25c-7f4483e41699","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:29.280160566Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:29.283895174Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e826bc23-4a5f-43b8-a25c-7f4483e41699","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:29.285187615Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e826bc23-4a5f-43b8-a25c-7f4483e41699","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:29.28539921Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:29.285445041Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:29.285481682Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:29.285516943Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:29.287159302Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:29.289887956Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:29.520691759Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"uri":"/crucible/0/regions","method":"POST","req_id":"3e8a9a77-5e21-45c8-9219-86f90a2b3ec5","remote_addr":"[fd00:1122:3344:101::b]:46751","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":59,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.247118018Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"uri":"/crucible/0/regions/23f3c2d4-c9f8-40af-9506-e644e1b2984d","method":"GET","req_id":"2b3fae9b-4635-48e8-981a-636e160cfb4a","remote_addr":"[fd00:1122:3344:101::b]:46751","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":66,"response_code":200}
{"msg":"path is \"/data/regions/23f3c2d4-c9f8-40af-9506-e644e1b2984d\"","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.253180921Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"datafile"}
{"msg":"dataset is oxp_6ee68421-c881-4fd9-b9ee-a17cfc057f47/crucible/regions/23f3c2d4-c9f8-40af-9506-e644e1b2984d","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.260279789Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"datafile"}
{"msg":"snapshot is oxp_6ee68421-c881-4fd9-b9ee-a17cfc057f47/crucible/regions/23f3c2d4-c9f8-40af-9506-e644e1b2984d@8988f26a-d638-46b9-aba4-7a85295caf98","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.269648391Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_6ee68421-c881-4fd9-b9ee-a17cfc057f47/crucible/regions/23f3c2d4-c9f8-40af-9506-e644e1b2984d\", \"8988f26a-d638-46b9-aba4-7a85295caf98\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.269690552Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.283042698Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"uri":"/crucible/0/regions/23f3c2d4-c9f8-40af-9506-e644e1b2984d/snapshots/8988f26a-d638-46b9-aba4-7a85295caf98","method":"GET","req_id":"aca88c24-b2ab-4068-b899-714bc1a23aa2","remote_addr":"[fd00:1122:3344:101::b]:46751","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":29886,"response_code":200}
{"msg":"path is \"/data/regions/23f3c2d4-c9f8-40af-9506-e644e1b2984d\"","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.287684688Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"datafile"}
{"msg":"dataset is oxp_6ee68421-c881-4fd9-b9ee-a17cfc057f47/crucible/regions/23f3c2d4-c9f8-40af-9506-e644e1b2984d","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.294845418Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"datafile"}
{"msg":"snapshot is oxp_6ee68421-c881-4fd9-b9ee-a17cfc057f47/crucible/regions/23f3c2d4-c9f8-40af-9506-e644e1b2984d@8988f26a-d638-46b9-aba4-7a85295caf98","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.300998453Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_6ee68421-c881-4fd9-b9ee-a17cfc057f47/crucible/regions/23f3c2d4-c9f8-40af-9506-e644e1b2984d\", \"8988f26a-d638-46b9-aba4-7a85295caf98\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.301032194Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 23f3c2d4-c9f8-40af-9506-e644e1b2984d-8988f26a-d638-46b9-aba4-7a85295caf98 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.30974323Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"datafile"}
{"msg":"applying SMF actions for region 23f3c2d4-c9f8-40af-9506-e644e1b2984d running snapshot 8988f26a-d638-46b9-aba4-7a85295caf98 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.31523315Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.315259461Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"uri":"/crucible/0/regions/23f3c2d4-c9f8-40af-9506-e644e1b2984d/snapshots/8988f26a-d638-46b9-aba4-7a85295caf98/run","method":"POST","req_id":"1f27b0fe-abf5-444b-a234-51fd93eb3837","remote_addr":"[fd00:1122:3344:101::b]:46751","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":27612,"response_code":200}
{"msg":"creating missing snapshot instance snapshot-23f3c2d4-c9f8-40af-9506-e644e1b2984d-8988f26a-d638-46b9-aba4-7a85295caf98","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.318306493Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-23f3c2d4-c9f8-40af-9506-e644e1b2984d-8988f26a-d638-46b9-aba4-7a85295caf98","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.325858902Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-23f3c2d4-c9f8-40af-9506-e644e1b2984d-8988f26a-d638-46b9-aba4-7a85295caf98","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.331118806Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/23f3c2d4-c9f8-40af-9506-e644e1b2984d/.zfs/snapshot/8988f26a-d638-46b9-aba4-7a85295caf98","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.331347891Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.331398043Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.331439374Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.331485015Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.331538886Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.333234026Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.336278648Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"worker"}
{"msg":"region 23f3c2d4-c9f8-40af-9506-e644e1b2984d running snapshot 8988f26a-d638-46b9-aba4-7a85295caf98 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.336297499Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"datafile"}
{"msg":"path is \"/data/regions/23f3c2d4-c9f8-40af-9506-e644e1b2984d\"","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.537843919Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"datafile"}
{"msg":"dataset is oxp_6ee68421-c881-4fd9-b9ee-a17cfc057f47/crucible/regions/23f3c2d4-c9f8-40af-9506-e644e1b2984d","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.545016039Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"datafile"}
{"msg":"snapshot is oxp_6ee68421-c881-4fd9-b9ee-a17cfc057f47/crucible/regions/23f3c2d4-c9f8-40af-9506-e644e1b2984d@8988f26a-d638-46b9-aba4-7a85295caf98","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.550453357Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_6ee68421-c881-4fd9-b9ee-a17cfc057f47/crucible/regions/23f3c2d4-c9f8-40af-9506-e644e1b2984d\", \"8988f26a-d638-46b9-aba4-7a85295caf98\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.550514169Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-19T07:43:31.557213067Z","hostname":"oxz_crucible_51415dca-2fc0-46fe-bd99-0a60228642c1","pid":23114,"uri":"/crucible/0/regions/23f3c2d4-c9f8-40af-9506-e644e1b2984d/snapshots/8988f26a-d638-46b9-aba4-7a85295caf98/run","method":"POST","req_id":"1ed6ed23-0023-4b9b-a99e-eeabf6ceebc0","remote_addr":"[fd00:1122:3344:101::b]:46751","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":19419,"response_code":200}