[ Jun 23 23:36:11 Enabled. ]
[ Jun 23 23:36:11 Rereading configuration. ]
[ Jun 23 23:36:12 Rereading configuration. ]
[ Jun 23 23:36:16 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_6263251a-e390-4b66-bf69-db518257072d/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: bd9a0e2abe6b6b89aec8c85f4ee57474144ed150\nCommit timestamp: 2026-05-14T15:48:00.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:16.210279827Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843}
{"msg":"dataset: \"oxp_6263251a-e390-4b66-bf69-db518257072d/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:16.210508502Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:16.210522772Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:16.210530212Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:16.225011967Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:16.289065645Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:42.821512637Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:58600"}
{"msg":"region b3ae3754-75f0-4700-a570-7eb960abb92f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:42.855039643Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:42.859875405Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:42.861480618Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"uri":"/crucible/0/regions","method":"POST","req_id":"8873fd32-289b-4b65-bbb7-9b61f72b0b8a","remote_addr":"[fd00:1122:3344:101::b]:58600","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":13422,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_6263251a-e390-4b66-bf69-db518257072d/crucible/regions/b3ae3754-75f0-4700-a570-7eb960abb92f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:42.868729291Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_6263251a-e390-4b66-bf69-db518257072d/crucible/regions/b3ae3754-75f0-4700-a570-7eb960abb92f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:42.868751401Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b3ae3754-75f0-4700-a570-7eb960abb92f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b3ae3754-75f0-4700-a570-7eb960abb92f\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:42.981233638Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"region":"b3ae3754-75f0-4700-a570-7eb960abb92f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b3ae3754-75f0-4700-a570-7eb960abb92f\" \"--data\" \"/data/regions/b3ae3754-75f0-4700-a570-7eb960abb92f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:42.98323946Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"region":"b3ae3754-75f0-4700-a570-7eb960abb92f","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:43.05971101Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"uri":"/crucible/0/regions","method":"POST","req_id":"4a2d3316-bf0d-4ff1-86f5-eaef0ab53a19","remote_addr":"[fd00:1122:3344:101::b]:58600","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":58,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:43.116901523Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"region":"b3ae3754-75f0-4700-a570-7eb960abb92f","component":"worker"}
{"msg":"region b3ae3754-75f0-4700-a570-7eb960abb92f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:43.116982904Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:43.11721644Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b3ae3754-75f0-4700-a570-7eb960abb92f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:43.119702972Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b3ae3754-75f0-4700-a570-7eb960abb92f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:43.125249509Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:43.139243993Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b3ae3754-75f0-4700-a570-7eb960abb92f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:43.142933891Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b3ae3754-75f0-4700-a570-7eb960abb92f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:43.143675176Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:43.143733167Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:43.143770468Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:43.143809319Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:43.157991687Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:43.163372441Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:43.500039704Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"uri":"/crucible/0/regions","method":"POST","req_id":"27637003-81d9-43a9-9a7c-72d6a47f13f7","remote_addr":"[fd00:1122:3344:101::b]:58600","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":89,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:44.352911569Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"uri":"/crucible/0/regions/b3ae3754-75f0-4700-a570-7eb960abb92f","method":"GET","req_id":"e905d43e-8c9f-4963-9945-eca79e58bcd8","remote_addr":"[fd00:1122:3344:101::b]:58600","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":56,"response_code":200}
{"msg":"path is \"/data/regions/b3ae3754-75f0-4700-a570-7eb960abb92f\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:44.356531745Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"component":"datafile"}
{"msg":"dataset is oxp_6263251a-e390-4b66-bf69-db518257072d/crucible/regions/b3ae3754-75f0-4700-a570-7eb960abb92f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:44.364336669Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"component":"datafile"}
{"msg":"region b3ae3754-75f0-4700-a570-7eb960abb92f state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:44.372002831Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:44.372223525Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:44.372286467Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"uri":"/crucible/0/regions/b3ae3754-75f0-4700-a570-7eb960abb92f","method":"DELETE","req_id":"85fe56ad-4b91-4ae5-818b-eb8967444a00","remote_addr":"[fd00:1122:3344:101::b]:58600","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":15748,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-b3ae3754-75f0-4700-a570-7eb960abb92f (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:44.373259527Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:44.375594596Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"uri":"/crucible/0/regions/b3ae3754-75f0-4700-a570-7eb960abb92f","method":"GET","req_id":"48214e6f-3a04-4cfd-9f51-9146ee06143a","remote_addr":"[fd00:1122:3344:101::b]:58600","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":53,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:44.378839154Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_6263251a-e390-4b66-bf69-db518257072d/crucible/regions/b3ae3754-75f0-4700-a570-7eb960abb92f\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:44.387406415Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"region":"b3ae3754-75f0-4700-a570-7eb960abb92f","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:44.438834667Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"uri":"/crucible/0/regions/b3ae3754-75f0-4700-a570-7eb960abb92f","method":"GET","req_id":"d555a4ed-d700-47d6-9d61-a55dadacf00d","remote_addr":"[fd00:1122:3344:101::b]:58600","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":51,"response_code":200}
{"msg":"region b3ae3754-75f0-4700-a570-7eb960abb92f state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:44.467836677Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-23T23:36:44.610769885Z","hostname":"oxz_crucible_57a0a451-b785-4985-bf8e-0a8b32404181","pid":24843,"uri":"/crucible/0/regions/b3ae3754-75f0-4700-a570-7eb960abb92f","method":"GET","req_id":"7ef46791-e45d-415d-8263-9c50f59a8d25","remote_addr":"[fd00:1122:3344:101::b]:58600","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":49,"response_code":200}