[ Feb 28 23:08:50 Enabled. ]
[ Feb 28 23:08:50 Rereading configuration. ]
[ Feb 28 23:08:55 Rereading configuration. ]
[ Feb 28 23:09:03 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_d43be9fe-82e2-4aaf-b9fa-0c9161803402/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_d43be9fe-82e2-4aaf-b9fa-0c9161803402/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:03.620033613Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:03.622854671Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:03.622865842Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:03.641421719Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:03.848528476Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:53.364574045Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:34847"}
{"msg":"region b694e8e8-3baf-41ec-9d44-1a7c51f7eb89 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:53.392461063Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:53.394059688Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:53.398262935Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"uri":"/crucible/0/regions","method":"POST","req_id":"aa237c68-11c1-44c3-85b5-00beae4babbf","remote_addr":"[fd00:1122:3344:101::a]:34847","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":16800,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_d43be9fe-82e2-4aaf-b9fa-0c9161803402/crucible/regions/b694e8e8-3baf-41ec-9d44-1a7c51f7eb89","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:53.405614Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_d43be9fe-82e2-4aaf-b9fa-0c9161803402/crucible/regions/b694e8e8-3baf-41ec-9d44-1a7c51f7eb89","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:53.4056367Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b694e8e8-3baf-41ec-9d44-1a7c51f7eb89\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b694e8e8-3baf-41ec-9d44-1a7c51f7eb89\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:53.46404796Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"region":"b694e8e8-3baf-41ec-9d44-1a7c51f7eb89","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b694e8e8-3baf-41ec-9d44-1a7c51f7eb89\" \"--data\" \"/data/regions/b694e8e8-3baf-41ec-9d44-1a7c51f7eb89\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:53.466237811Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"region":"b694e8e8-3baf-41ec-9d44-1a7c51f7eb89","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:53.598802858Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"region":"b694e8e8-3baf-41ec-9d44-1a7c51f7eb89","component":"worker"}
{"msg":"region b694e8e8-3baf-41ec-9d44-1a7c51f7eb89 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:53.59887479Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:53.598958453Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b694e8e8-3baf-41ec-9d44-1a7c51f7eb89","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:53.599384345Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b694e8e8-3baf-41ec-9d44-1a7c51f7eb89","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:53.605235648Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:53.621020658Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b694e8e8-3baf-41ec-9d44-1a7c51f7eb89","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:53.625191515Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b694e8e8-3baf-41ec-9d44-1a7c51f7eb89","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:53.625904194Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:53.625971976Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:53.626010267Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:53.626063779Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:53.630271416Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:53.643000131Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:53.647217269Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"uri":"/crucible/0/regions","method":"POST","req_id":"75543d03-5d5c-4bd2-9566-b69a6e599d84","remote_addr":"[fd00:1122:3344:101::a]:34847","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":61,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:54.43782133Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"uri":"/crucible/0/regions/b694e8e8-3baf-41ec-9d44-1a7c51f7eb89","method":"GET","req_id":"d4aa28c7-78fc-4ceb-8bf5-c3ead23b70d9","remote_addr":"[fd00:1122:3344:101::a]:34847","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":2069,"response_code":"200"}
{"msg":"path is \"/data/regions/b694e8e8-3baf-41ec-9d44-1a7c51f7eb89\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:54.440570197Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"component":"datafile"}
{"msg":"dataset is oxp_d43be9fe-82e2-4aaf-b9fa-0c9161803402/crucible/regions/b694e8e8-3baf-41ec-9d44-1a7c51f7eb89","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:54.451332837Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"component":"datafile"}
{"msg":"region b694e8e8-3baf-41ec-9d44-1a7c51f7eb89 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:54.461102489Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:54.461241313Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:54.461302395Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"uri":"/crucible/0/regions/b694e8e8-3baf-41ec-9d44-1a7c51f7eb89","method":"DELETE","req_id":"821733bb-9e28-405f-bffe-a484d4461aba","remote_addr":"[fd00:1122:3344:101::a]:34847","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":20755,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-b694e8e8-3baf-41ec-9d44-1a7c51f7eb89 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:54.462299223Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:54.464625058Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"uri":"/crucible/0/regions/b694e8e8-3baf-41ec-9d44-1a7c51f7eb89","method":"GET","req_id":"d456e49d-ba6c-438b-a55b-728c325580c5","remote_addr":"[fd00:1122:3344:101::a]:34847","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":47,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:54.468636269Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_d43be9fe-82e2-4aaf-b9fa-0c9161803402/crucible/regions/b694e8e8-3baf-41ec-9d44-1a7c51f7eb89\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:54.479929484Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"region":"b694e8e8-3baf-41ec-9d44-1a7c51f7eb89","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:54.527290825Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"uri":"/crucible/0/regions/b694e8e8-3baf-41ec-9d44-1a7c51f7eb89","method":"GET","req_id":"ad54d0bc-a910-445a-8d96-2abd8c22ef56","remote_addr":"[fd00:1122:3344:101::a]:34847","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":60,"response_code":"200"}
{"msg":"region b694e8e8-3baf-41ec-9d44-1a7c51f7eb89 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:54.542484609Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T23:09:54.691604199Z","hostname":"oxz_crucible_7a11c084-2890-4137-9a8e-e027a350cf19","pid":22786,"uri":"/crucible/0/regions/b694e8e8-3baf-41ec-9d44-1a7c51f7eb89","method":"GET","req_id":"627f3b6f-5d06-4214-b9e8-c4abdba9894b","remote_addr":"[fd00:1122:3344:101::a]:34847","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":29,"response_code":"200"}