[ Apr 14 23:30:37 Enabled. ]
[ Apr 14 23:30:37 Rereading configuration. ]
[ Apr 14 23:30:39 Rereading configuration. ]
[ Apr 14 23:30:46 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_e86a9308-4bb3-4f15-942b-75f1ecaddb2a/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:30:46.811194504Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631}
{"msg":"dataset: \"oxp_e86a9308-4bb3-4f15-942b-75f1ecaddb2a/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:30:46.812002807Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:30:46.812016897Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:30:46.812024637Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:30:46.8238888Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:30:46.910331895Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:32.713132964Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:45989"}
{"msg":"region eb747e90-5143-4e76-bcce-f1b1564f40c2 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:32.740899031Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:32.744006936Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:32.746903755Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"uri":"/crucible/0/regions","method":"POST","req_id":"6f206e46-5722-4d92-ad5f-766953231e3b","remote_addr":"[fd00:1122:3344:101::c]:45989","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":16902,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_e86a9308-4bb3-4f15-942b-75f1ecaddb2a/crucible/regions/eb747e90-5143-4e76-bcce-f1b1564f40c2","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:32.753185755Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_e86a9308-4bb3-4f15-942b-75f1ecaddb2a/crucible/regions/eb747e90-5143-4e76-bcce-f1b1564f40c2","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:32.753204486Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"eb747e90-5143-4e76-bcce-f1b1564f40c2\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/eb747e90-5143-4e76-bcce-f1b1564f40c2\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:32.847017132Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"region":"eb747e90-5143-4e76-bcce-f1b1564f40c2","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"eb747e90-5143-4e76-bcce-f1b1564f40c2\" \"--data\" \"/data/regions/eb747e90-5143-4e76-bcce-f1b1564f40c2\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:32.850763914Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"region":"eb747e90-5143-4e76-bcce-f1b1564f40c2","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:32.966419934Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"uri":"/crucible/0/regions","method":"POST","req_id":"2b25d6e5-36d0-48d8-ac3c-12ce2701a5c8","remote_addr":"[fd00:1122:3344:101::c]:45989","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":58,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.200166111Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"region":"eb747e90-5143-4e76-bcce-f1b1564f40c2","component":"worker"}
{"msg":"region eb747e90-5143-4e76-bcce-f1b1564f40c2 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.200242984Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.200510461Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-eb747e90-5143-4e76-bcce-f1b1564f40c2","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.20083747Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-eb747e90-5143-4e76-bcce-f1b1564f40c2","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.20742856Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.223710913Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-eb747e90-5143-4e76-bcce-f1b1564f40c2","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.22656007Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/eb747e90-5143-4e76-bcce-f1b1564f40c2","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.22726862Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.227331872Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.227371493Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.227403893Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.232261956Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.236983765Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.430594649Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"uri":"/crucible/0/regions","method":"POST","req_id":"e9018cd2-8442-4767-8f2d-f72f128713c5","remote_addr":"[fd00:1122:3344:101::c]:45989","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":55,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.795811908Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"uri":"/crucible/0/regions/eb747e90-5143-4e76-bcce-f1b1564f40c2","method":"GET","req_id":"e5aed6a6-0c7e-4641-9f8c-01158908b6a5","remote_addr":"[fd00:1122:3344:101::c]:45989","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":77,"response_code":200}
{"msg":"path is \"/data/regions/eb747e90-5143-4e76-bcce-f1b1564f40c2\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.798610864Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"datafile"}
{"msg":"dataset is oxp_e86a9308-4bb3-4f15-942b-75f1ecaddb2a/crucible/regions/eb747e90-5143-4e76-bcce-f1b1564f40c2","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.807465625Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"datafile"}
{"msg":"region eb747e90-5143-4e76-bcce-f1b1564f40c2 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.815058232Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.815147784Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.815164765Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"uri":"/crucible/0/regions/eb747e90-5143-4e76-bcce-f1b1564f40c2","method":"DELETE","req_id":"70bcb8c7-f94b-4da6-9104-94b584bdf823","remote_addr":"[fd00:1122:3344:101::c]:45989","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":16554,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-eb747e90-5143-4e76-bcce-f1b1564f40c2 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.816166382Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.818198488Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"uri":"/crucible/0/regions/eb747e90-5143-4e76-bcce-f1b1564f40c2","method":"GET","req_id":"c8701714-6c31-47fb-861a-da9ad6c50048","remote_addr":"[fd00:1122:3344:101::c]:45989","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":29,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.821193369Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_e86a9308-4bb3-4f15-942b-75f1ecaddb2a/crucible/regions/eb747e90-5143-4e76-bcce-f1b1564f40c2\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.829148956Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"region":"eb747e90-5143-4e76-bcce-f1b1564f40c2","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.875301633Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"uri":"/crucible/0/regions/eb747e90-5143-4e76-bcce-f1b1564f40c2","method":"GET","req_id":"e2ec6a59-7d12-4c52-b596-232af0d51ef6","remote_addr":"[fd00:1122:3344:101::c]:45989","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":33,"response_code":200}
{"msg":"region eb747e90-5143-4e76-bcce-f1b1564f40c2 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.884059542Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:33.98531466Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"uri":"/crucible/0/regions/eb747e90-5143-4e76-bcce-f1b1564f40c2","method":"GET","req_id":"67829584-cc67-4530-9895-626158bf7490","remote_addr":"[fd00:1122:3344:101::c]:45989","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":27,"response_code":200}
{"msg":"region d3dda9a4-62eb-40e0-9f7d-3c98413576d6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.275900908Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.276048182Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.276071313Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"uri":"/crucible/0/regions","method":"POST","req_id":"210d2ac0-649e-4420-b364-4997f32022b7","remote_addr":"[fd00:1122:3344:101::c]:45989","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":195,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_e86a9308-4bb3-4f15-942b-75f1ecaddb2a/crucible/regions/d3dda9a4-62eb-40e0-9f7d-3c98413576d6","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.281942853Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_e86a9308-4bb3-4f15-942b-75f1ecaddb2a/crucible/regions/d3dda9a4-62eb-40e0-9f7d-3c98413576d6","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.281984124Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"d3dda9a4-62eb-40e0-9f7d-3c98413576d6\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/d3dda9a4-62eb-40e0-9f7d-3c98413576d6\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.335270626Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"region":"d3dda9a4-62eb-40e0-9f7d-3c98413576d6","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"d3dda9a4-62eb-40e0-9f7d-3c98413576d6\" \"--data\" \"/data/regions/d3dda9a4-62eb-40e0-9f7d-3c98413576d6\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.335309307Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"region":"d3dda9a4-62eb-40e0-9f7d-3c98413576d6","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.411160443Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"region":"d3dda9a4-62eb-40e0-9f7d-3c98413576d6","component":"worker"}
{"msg":"region d3dda9a4-62eb-40e0-9f7d-3c98413576d6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.411187844Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.411281656Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-d3dda9a4-62eb-40e0-9f7d-3c98413576d6","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.412050677Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-d3dda9a4-62eb-40e0-9f7d-3c98413576d6","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.415604334Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.423026766Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-d3dda9a4-62eb-40e0-9f7d-3c98413576d6","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.42609344Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d3dda9a4-62eb-40e0-9f7d-3c98413576d6","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.426373128Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.426413538Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.42645181Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.426484871Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.429501833Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.434228671Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.585689277Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"uri":"/crucible/0/regions","method":"POST","req_id":"2645f865-1213-4779-92e0-bd6456110a5e","remote_addr":"[fd00:1122:3344:101::c]:45989","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":56,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:13.793422114Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:53373"}
{"msg":"region 5480d811-7072-4eb3-8510-83a344f26a59 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:13.795939633Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:13.795972964Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:13.795981434Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"uri":"/crucible/0/regions","method":"POST","req_id":"7b9496ec-68e8-4bbf-9cdc-a1a4a350de90","remote_addr":"[fd00:1122:3344:101::c]:53373","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":303,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_e86a9308-4bb3-4f15-942b-75f1ecaddb2a/crucible/regions/5480d811-7072-4eb3-8510-83a344f26a59","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:13.803766706Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_e86a9308-4bb3-4f15-942b-75f1ecaddb2a/crucible/regions/5480d811-7072-4eb3-8510-83a344f26a59","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:13.803784396Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"5480d811-7072-4eb3-8510-83a344f26a59\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/5480d811-7072-4eb3-8510-83a344f26a59\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:13.89605155Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"region":"5480d811-7072-4eb3-8510-83a344f26a59","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"5480d811-7072-4eb3-8510-83a344f26a59\" \"--data\" \"/data/regions/5480d811-7072-4eb3-8510-83a344f26a59\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:13.896094131Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"region":"5480d811-7072-4eb3-8510-83a344f26a59","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:14.098103064Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"region":"5480d811-7072-4eb3-8510-83a344f26a59","component":"worker"}
{"msg":"region 5480d811-7072-4eb3-8510-83a344f26a59 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:14.098137605Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:14.098253978Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-5480d811-7072-4eb3-8510-83a344f26a59","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:14.099091981Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-5480d811-7072-4eb3-8510-83a344f26a59","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:14.126631221Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:14.16882224Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"uri":"/crucible/0/regions","method":"POST","req_id":"a2927c61-e922-4013-be52-6e7cb0ba0c9b","remote_addr":"[fd00:1122:3344:101::c]:53373","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":58,"response_code":200}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:14.174025672Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5480d811-7072-4eb3-8510-83a344f26a59","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:14.189703819Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5480d811-7072-4eb3-8510-83a344f26a59","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:14.190038088Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:14.19009327Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:14.190130171Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:14.190162992Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:14.206404134Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:14.240134933Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.799133172Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"uri":"/crucible/0/regions/d3dda9a4-62eb-40e0-9f7d-3c98413576d6","method":"GET","req_id":"c65b576b-6443-4dd0-a61c-bdb4206b1d05","remote_addr":"[fd00:1122:3344:101::c]:53373","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":52,"response_code":200}
{"msg":"path is \"/data/regions/d3dda9a4-62eb-40e0-9f7d-3c98413576d6\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.802473153Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"datafile"}
{"msg":"dataset is oxp_e86a9308-4bb3-4f15-942b-75f1ecaddb2a/crucible/regions/d3dda9a4-62eb-40e0-9f7d-3c98413576d6","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.808438276Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"datafile"}
{"msg":"snapshot is oxp_e86a9308-4bb3-4f15-942b-75f1ecaddb2a/crucible/regions/d3dda9a4-62eb-40e0-9f7d-3c98413576d6@18ff68e4-f03f-48ec-8b7a-dfb1c4879220","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.81485317Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_e86a9308-4bb3-4f15-942b-75f1ecaddb2a/crucible/regions/d3dda9a4-62eb-40e0-9f7d-3c98413576d6\", \"18ff68e4-f03f-48ec-8b7a-dfb1c4879220\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.814878431Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.828819231Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"uri":"/crucible/0/regions/d3dda9a4-62eb-40e0-9f7d-3c98413576d6/snapshots/18ff68e4-f03f-48ec-8b7a-dfb1c4879220","method":"GET","req_id":"a5d02d5e-3bab-44c1-a5d7-3a8c180c8f6c","remote_addr":"[fd00:1122:3344:101::c]:53373","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":26373,"response_code":200}
{"msg":"path is \"/data/regions/d3dda9a4-62eb-40e0-9f7d-3c98413576d6\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.832624724Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"datafile"}
{"msg":"dataset is oxp_e86a9308-4bb3-4f15-942b-75f1ecaddb2a/crucible/regions/d3dda9a4-62eb-40e0-9f7d-3c98413576d6","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.839392919Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"datafile"}
{"msg":"snapshot is oxp_e86a9308-4bb3-4f15-942b-75f1ecaddb2a/crucible/regions/d3dda9a4-62eb-40e0-9f7d-3c98413576d6@18ff68e4-f03f-48ec-8b7a-dfb1c4879220","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.845053293Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_e86a9308-4bb3-4f15-942b-75f1ecaddb2a/crucible/regions/d3dda9a4-62eb-40e0-9f7d-3c98413576d6\", \"18ff68e4-f03f-48ec-8b7a-dfb1c4879220\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.845069414Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot d3dda9a4-62eb-40e0-9f7d-3c98413576d6-18ff68e4-f03f-48ec-8b7a-dfb1c4879220 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.857397509Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"datafile"}
{"msg":"applying SMF actions for region d3dda9a4-62eb-40e0-9f7d-3c98413576d6 running snapshot 18ff68e4-f03f-48ec-8b7a-dfb1c4879220 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.857630506Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.857646186Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"uri":"/crucible/0/regions/d3dda9a4-62eb-40e0-9f7d-3c98413576d6/snapshots/18ff68e4-f03f-48ec-8b7a-dfb1c4879220/run","method":"POST","req_id":"b19929dc-719a-4396-8f36-b0ac07cc2a7e","remote_addr":"[fd00:1122:3344:101::c]:53373","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":25015,"response_code":200}
{"msg":"creating missing snapshot instance snapshot-d3dda9a4-62eb-40e0-9f7d-3c98413576d6-18ff68e4-f03f-48ec-8b7a-dfb1c4879220","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.862664433Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-d3dda9a4-62eb-40e0-9f7d-3c98413576d6-18ff68e4-f03f-48ec-8b7a-dfb1c4879220","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.872513321Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-d3dda9a4-62eb-40e0-9f7d-3c98413576d6-18ff68e4-f03f-48ec-8b7a-dfb1c4879220","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.907211656Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d3dda9a4-62eb-40e0-9f7d-3c98413576d6/.zfs/snapshot/18ff68e4-f03f-48ec-8b7a-dfb1c4879220","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.907507584Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.907553976Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.907589517Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.907626628Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.907674609Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.920278512Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.935577929Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"worker"}
{"msg":"region d3dda9a4-62eb-40e0-9f7d-3c98413576d6 running snapshot 18ff68e4-f03f-48ec-8b7a-dfb1c4879220 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.935588499Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"datafile"}
{"msg":"path is \"/data/regions/d3dda9a4-62eb-40e0-9f7d-3c98413576d6\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:15.996133509Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"datafile"}
{"msg":"dataset is oxp_e86a9308-4bb3-4f15-942b-75f1ecaddb2a/crucible/regions/d3dda9a4-62eb-40e0-9f7d-3c98413576d6","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.002373059Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"datafile"}
{"msg":"snapshot is oxp_e86a9308-4bb3-4f15-942b-75f1ecaddb2a/crucible/regions/d3dda9a4-62eb-40e0-9f7d-3c98413576d6@18ff68e4-f03f-48ec-8b7a-dfb1c4879220","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.008631369Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_e86a9308-4bb3-4f15-942b-75f1ecaddb2a/crucible/regions/d3dda9a4-62eb-40e0-9f7d-3c98413576d6\", \"18ff68e4-f03f-48ec-8b7a-dfb1c4879220\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.00866356Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.016143994Z","hostname":"oxz_crucible_da4122c9-48c3-48d0-baa0-77af0b5a1af4","pid":28631,"uri":"/crucible/0/regions/d3dda9a4-62eb-40e0-9f7d-3c98413576d6/snapshots/18ff68e4-f03f-48ec-8b7a-dfb1c4879220/run","method":"POST","req_id":"bb81d4d1-8c62-4757-a359-d05b4b0ec668","remote_addr":"[fd00:1122:3344:101::c]:53373","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":20044,"response_code":200}