[ Nov 25 19:31:02 Enabled. ]
[ Nov 25 19:31:02 Rereading configuration. ]
[ Nov 25 19:31:05 Rereading configuration. ]
[ Nov 25 19:31:09 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_d7af1128-7206-4198-91cb-ef632ea8556e/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_d7af1128-7206-4198-91cb-ef632ea8556e/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:10.026484861Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:10.030190424Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:10.030204054Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:10.046926672Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:10.106719925Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.552778293Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:40842"}
{"msg":"region 7ceeb083-236e-4c0d-904b-8aec992d4977 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.582653119Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.587511634Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"uri":"/crucible/0/regions","method":"POST","req_id":"29002d94-ddc3-48ed-9eb7-bf21781b6564","remote_addr":"[fd00:1122:3344:101::b]:40842","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":10728,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.587536144Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_d7af1128-7206-4198-91cb-ef632ea8556e/crucible/regions/7ceeb083-236e-4c0d-904b-8aec992d4977","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.596316087Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_d7af1128-7206-4198-91cb-ef632ea8556e/crucible/regions/7ceeb083-236e-4c0d-904b-8aec992d4977","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.596378968Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"7ceeb083-236e-4c0d-904b-8aec992d4977\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/7ceeb083-236e-4c0d-904b-8aec992d4977\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.683304023Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"region":"7ceeb083-236e-4c0d-904b-8aec992d4977","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"7ceeb083-236e-4c0d-904b-8aec992d4977\" \"--data\" \"/data/regions/7ceeb083-236e-4c0d-904b-8aec992d4977\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.685304043Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"region":"7ceeb083-236e-4c0d-904b-8aec992d4977","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.77691635Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"uri":"/crucible/0/regions","method":"POST","req_id":"5b390298-e6b5-41cc-89ed-591fb7cf34ec","remote_addr":"[fd00:1122:3344:101::b]:40842","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":74,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.852482693Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"region":"7ceeb083-236e-4c0d-904b-8aec992d4977","component":"worker"}
{"msg":"region 7ceeb083-236e-4c0d-904b-8aec992d4977 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.852547884Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.852759818Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-7ceeb083-236e-4c0d-904b-8aec992d4977","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.853157496Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-7ceeb083-236e-4c0d-904b-8aec992d4977","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.858029762Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.882179485Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-7ceeb083-236e-4c0d-904b-8aec992d4977","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.887699564Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7ceeb083-236e-4c0d-904b-8aec992d4977","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.888282595Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.888348857Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.888391837Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.888436998Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.893169141Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:40.899020766Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.528549267Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"uri":"/crucible/0/regions","method":"POST","req_id":"027f63ef-52f0-4029-bcef-6f7b685082fc","remote_addr":"[fd00:1122:3344:101::b]:40842","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":61,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.829477891Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"uri":"/crucible/0/regions/7ceeb083-236e-4c0d-904b-8aec992d4977","method":"GET","req_id":"2d63efb7-a5e8-473d-8925-098f3a9504c3","remote_addr":"[fd00:1122:3344:101::b]:40842","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":73,"response_code":"200"}
{"msg":"path is \"/data/regions/7ceeb083-236e-4c0d-904b-8aec992d4977\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.832051802Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"component":"datafile"}
{"msg":"dataset is oxp_d7af1128-7206-4198-91cb-ef632ea8556e/crucible/regions/7ceeb083-236e-4c0d-904b-8aec992d4977","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.843659689Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"component":"datafile"}
{"msg":"region 7ceeb083-236e-4c0d-904b-8aec992d4977 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.853420701Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.853649185Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.853720067Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"uri":"/crucible/0/regions/7ceeb083-236e-4c0d-904b-8aec992d4977","method":"DELETE","req_id":"065bea72-e693-4df8-81b6-2b78864d1f84","remote_addr":"[fd00:1122:3344:101::b]:40842","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":21691,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-7ceeb083-236e-4c0d-904b-8aec992d4977 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.856703245Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.857007731Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"uri":"/crucible/0/regions/7ceeb083-236e-4c0d-904b-8aec992d4977","method":"GET","req_id":"37407595-a2a1-4d47-890e-3d4cf184a259","remote_addr":"[fd00:1122:3344:101::b]:40842","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":30,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.861652113Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_d7af1128-7206-4198-91cb-ef632ea8556e/crucible/regions/7ceeb083-236e-4c0d-904b-8aec992d4977\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.872338862Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"region":"7ceeb083-236e-4c0d-904b-8aec992d4977","component":"worker"}
{"msg":"region 7ceeb083-236e-4c0d-904b-8aec992d4977 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.947081418Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:41.962372218Z","hostname":"oxz_crucible_863e78e3-a753-4fb2-97de-c46023f03292","pid":28511,"uri":"/crucible/0/regions/7ceeb083-236e-4c0d-904b-8aec992d4977","method":"GET","req_id":"7ed61e1f-1d0f-4073-bdc8-a4d5f95be5e4","remote_addr":"[fd00:1122:3344:101::b]:40842","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":93,"response_code":"200"}