[ Mar 16 22:27:38 Enabled. ]
[ Mar 16 22:27:38 Rereading configuration. ]
[ Mar 16 22:27:40 Rereading configuration. ]
[ Mar 16 22:27:44 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_a51b3b05-7403-4806-9065-c4a139ceb59c/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:27:45.093535382Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278}
{"msg":"dataset: \"oxp_a51b3b05-7403-4806-9065-c4a139ceb59c/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:27:45.094226557Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:27:45.094239567Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:27:45.094246957Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:27:45.109423129Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:27:45.183630134Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:38.95857155Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:53251"}
{"msg":"region 2b768871-623b-4154-ac3d-746a2eeca584 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:38.982518848Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:38.985296317Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:38.985341088Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"uri":"/crucible/0/regions","method":"POST","req_id":"d840b99f-31a6-46f4-a3bb-4e6b24e2a439","remote_addr":"[fd00:1122:3344:101::b]:53251","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":13367,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_a51b3b05-7403-4806-9065-c4a139ceb59c/crucible/regions/2b768871-623b-4154-ac3d-746a2eeca584","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:38.993277796Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_a51b3b05-7403-4806-9065-c4a139ceb59c/crucible/regions/2b768871-623b-4154-ac3d-746a2eeca584","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:38.993294227Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"2b768871-623b-4154-ac3d-746a2eeca584\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/2b768871-623b-4154-ac3d-746a2eeca584\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:39.070700969Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"region":"2b768871-623b-4154-ac3d-746a2eeca584","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"2b768871-623b-4154-ac3d-746a2eeca584\" \"--data\" \"/data/regions/2b768871-623b-4154-ac3d-746a2eeca584\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:39.072797834Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"region":"2b768871-623b-4154-ac3d-746a2eeca584","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:39.250280539Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"uri":"/crucible/0/regions","method":"POST","req_id":"8a380cf4-8706-4553-ae22-83e2499a4301","remote_addr":"[fd00:1122:3344:101::b]:53251","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":61,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:39.432556067Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"region":"2b768871-623b-4154-ac3d-746a2eeca584","component":"worker"}
{"msg":"region 2b768871-623b-4154-ac3d-746a2eeca584 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:39.432634679Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:39.432817032Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-2b768871-623b-4154-ac3d-746a2eeca584","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:39.43313939Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-2b768871-623b-4154-ac3d-746a2eeca584","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:39.444273246Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:39.469648634Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2b768871-623b-4154-ac3d-746a2eeca584","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:39.478327328Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2b768871-623b-4154-ac3d-746a2eeca584","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:39.478914871Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:39.478983432Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:39.479030713Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:39.479061004Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:39.484861957Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:39.496755049Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:39.637924155Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"uri":"/crucible/0/regions","method":"POST","req_id":"52e0498c-8b1f-4e23-838e-e8ae8ec3f2be","remote_addr":"[fd00:1122:3344:101::b]:53251","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":66,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:40.414066713Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"uri":"/crucible/0/regions/2b768871-623b-4154-ac3d-746a2eeca584","method":"GET","req_id":"b70ca009-fd22-4a1e-8836-86f1f5aa9149","remote_addr":"[fd00:1122:3344:101::b]:53251","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":83,"response_code":200}
{"msg":"path is \"/data/regions/2b768871-623b-4154-ac3d-746a2eeca584\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:40.416431263Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"component":"datafile"}
{"msg":"dataset is oxp_a51b3b05-7403-4806-9065-c4a139ceb59c/crucible/regions/2b768871-623b-4154-ac3d-746a2eeca584","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:40.424826561Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"component":"datafile"}
{"msg":"region 2b768871-623b-4154-ac3d-746a2eeca584 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:40.432261239Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:40.432444443Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"uri":"/crucible/0/regions/2b768871-623b-4154-ac3d-746a2eeca584","method":"DELETE","req_id":"9a4ed5b0-d934-43ad-b5d6-0bf23180c044","remote_addr":"[fd00:1122:3344:101::b]:53251","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":16025,"response_code":204}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:40.432501134Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-2b768871-623b-4154-ac3d-746a2eeca584 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:40.433398434Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:40.435518759Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"uri":"/crucible/0/regions/2b768871-623b-4154-ac3d-746a2eeca584","method":"GET","req_id":"cdbba6e3-b685-4f4d-8ba9-38014af19245","remote_addr":"[fd00:1122:3344:101::b]:53251","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":27,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:40.441606328Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_a51b3b05-7403-4806-9065-c4a139ceb59c/crucible/regions/2b768871-623b-4154-ac3d-746a2eeca584\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:40.449458534Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"region":"2b768871-623b-4154-ac3d-746a2eeca584","component":"worker"}
{"msg":"region 2b768871-623b-4154-ac3d-746a2eeca584 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:40.539361852Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T22:28:40.582947337Z","hostname":"oxz_crucible_c413109b-960b-49fa-bf29-49e9d6e2ce70","pid":23278,"uri":"/crucible/0/regions/2b768871-623b-4154-ac3d-746a2eeca584","method":"GET","req_id":"f68953f5-5c58-426c-b3e3-5fbb104df304","remote_addr":"[fd00:1122:3344:101::b]:53251","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":52,"response_code":200}