[ Jul 29 01:13:31 Enabled. ]
[ Jul 29 01:13:31 Rereading configuration. ]
[ Jul 29 01:13:33 Rereading configuration. ]
[ Jul 29 01:13:36 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_bf8f735b-7329-47db-b196-98c5bcddd430/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 2bfe090eb5318ec8c467157018db2429d4df535b\nCommit timestamp: 2026-06-25T19:33:01.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:13:37.008726365Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339}
{"msg":"dataset: \"oxp_bf8f735b-7329-47db-b196-98c5bcddd430/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:13:37.008901088Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:13:37.008912729Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:13:37.008918209Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:13:37.025183366Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:13:37.088604008Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:14:59.05704753Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:43230"}
{"msg":"region d40fb10f-2efe-4372-9909-ddb70d9b3983 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:14:59.086712425Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:14:59.091499367Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:14:59.09400849Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"uri":"/crucible/0/regions","method":"POST","req_id":"abebe101-5518-4fb7-8dce-8265ae1d945c","remote_addr":"[fd00:1122:3344:101::a]:43230","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":16489,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_bf8f735b-7329-47db-b196-98c5bcddd430/crucible/regions/d40fb10f-2efe-4372-9909-ddb70d9b3983","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:14:59.099237319Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_bf8f735b-7329-47db-b196-98c5bcddd430/crucible/regions/d40fb10f-2efe-4372-9909-ddb70d9b3983","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:14:59.099260919Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"d40fb10f-2efe-4372-9909-ddb70d9b3983\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/d40fb10f-2efe-4372-9909-ddb70d9b3983\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:14:59.14844288Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"region":"d40fb10f-2efe-4372-9909-ddb70d9b3983","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"d40fb10f-2efe-4372-9909-ddb70d9b3983\" \"--data\" \"/data/regions/d40fb10f-2efe-4372-9909-ddb70d9b3983\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:14:59.150470943Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"region":"d40fb10f-2efe-4372-9909-ddb70d9b3983","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:14:59.27466876Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"region":"d40fb10f-2efe-4372-9909-ddb70d9b3983","component":"worker"}
{"msg":"region d40fb10f-2efe-4372-9909-ddb70d9b3983 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:14:59.27876927Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:14:59.278910622Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-d40fb10f-2efe-4372-9909-ddb70d9b3983","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:14:59.281075939Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-d40fb10f-2efe-4372-9909-ddb70d9b3983","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:14:59.288785231Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:14:59.3022532Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-d40fb10f-2efe-4372-9909-ddb70d9b3983","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:14:59.305524156Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d40fb10f-2efe-4372-9909-ddb70d9b3983","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:14:59.30635405Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:14:59.308625659Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:14:59.3086662Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:14:59.308703821Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:14:59.319068527Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:14:59.323627205Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:14:59.363699578Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"uri":"/crucible/0/regions","method":"POST","req_id":"e8818f80-6331-457e-96c8-c83017b69553","remote_addr":"[fd00:1122:3344:101::a]:43230","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":72,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:15:00.264870055Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"uri":"/crucible/0/regions/d40fb10f-2efe-4372-9909-ddb70d9b3983","method":"GET","req_id":"849ec111-274f-4f81-9971-adfadb58b138","remote_addr":"[fd00:1122:3344:101::a]:43230","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":70,"response_code":200}
{"msg":"path is \"/data/regions/d40fb10f-2efe-4372-9909-ddb70d9b3983\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:15:00.269393873Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"component":"datafile"}
{"msg":"dataset is oxp_bf8f735b-7329-47db-b196-98c5bcddd430/crucible/regions/d40fb10f-2efe-4372-9909-ddb70d9b3983","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:15:00.27921042Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"component":"datafile"}
{"msg":"region d40fb10f-2efe-4372-9909-ddb70d9b3983 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:15:00.287172876Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:15:00.287322798Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:15:00.287343369Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"uri":"/crucible/0/regions/d40fb10f-2efe-4372-9909-ddb70d9b3983","method":"DELETE","req_id":"8c521191-587b-4257-bd6b-e56c352a943e","remote_addr":"[fd00:1122:3344:101::a]:43230","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":18017,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-d40fb10f-2efe-4372-9909-ddb70d9b3983 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:15:00.288298455Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:15:00.291622152Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"uri":"/crucible/0/regions/d40fb10f-2efe-4372-9909-ddb70d9b3983","method":"GET","req_id":"b4795a9d-5069-4489-80f1-173b3fc60a7f","remote_addr":"[fd00:1122:3344:101::a]:43230","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":57,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:15:00.293130597Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_bf8f735b-7329-47db-b196-98c5bcddd430/crucible/regions/d40fb10f-2efe-4372-9909-ddb70d9b3983\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:15:00.302807062Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"region":"d40fb10f-2efe-4372-9909-ddb70d9b3983","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:15:00.350302022Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"uri":"/crucible/0/regions/d40fb10f-2efe-4372-9909-ddb70d9b3983","method":"GET","req_id":"da216046-edb9-4c7b-9f73-3d9762126fa0","remote_addr":"[fd00:1122:3344:101::a]:43230","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":55,"response_code":200}
{"msg":"region d40fb10f-2efe-4372-9909-ddb70d9b3983 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:15:00.376766614Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-29T01:15:00.470184427Z","hostname":"oxz_crucible_b86b41b7-a8e6-46a1-90b2-f50a62787621","pid":24339,"uri":"/crucible/0/regions/d40fb10f-2efe-4372-9909-ddb70d9b3983","method":"GET","req_id":"4ab4fc9b-e999-4e52-bb7e-713daed9856b","remote_addr":"[fd00:1122:3344:101::a]:43230","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":53,"response_code":200}