[ Oct  4 22:13:47 Enabled. ]
[ Oct  4 22:13:47 Rereading configuration. ]
[ Oct  4 22:13:48 Rereading configuration. ]
[ Oct  4 22:13:51 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_d4135165-e105-4373-9d12-4efe0b6e4c50/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_d4135165-e105-4373-9d12-4efe0b6e4c50/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:13:51.296430796Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:13:51.299942826Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:13:51.299957747Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:13:51.439984954Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:13:51.619983969Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:46.879045331Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:51333"}
{"msg":"region 3e0023dc-8018-43af-ac86-531375252051 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:46.909989069Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:46.914841326Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"uri":"/crucible/0/regions","method":"POST","req_id":"4cf8c71a-322a-4019-839a-ce4dcc42e64b","remote_addr":"[fd00:1122:3344:101::c]:51333","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":12664,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:46.915926418Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_d4135165-e105-4373-9d12-4efe0b6e4c50/crucible/regions/3e0023dc-8018-43af-ac86-531375252051","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:46.925424138Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_d4135165-e105-4373-9d12-4efe0b6e4c50/crucible/regions/3e0023dc-8018-43af-ac86-531375252051","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:46.925449388Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"3e0023dc-8018-43af-ac86-531375252051\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3e0023dc-8018-43af-ac86-531375252051\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:46.993443526Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"region":"3e0023dc-8018-43af-ac86-531375252051","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3e0023dc-8018-43af-ac86-531375252051\" \"--data\" \"/data/regions/3e0023dc-8018-43af-ac86-531375252051\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:46.997233442Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"region":"3e0023dc-8018-43af-ac86-531375252051","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.132603976Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"uri":"/crucible/0/regions","method":"POST","req_id":"8985f7d1-8e0d-4a76-b747-1ce582913cca","remote_addr":"[fd00:1122:3344:101::c]:51333","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":726,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.137985264Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"region":"3e0023dc-8018-43af-ac86-531375252051","component":"worker"}
{"msg":"region 3e0023dc-8018-43af-ac86-531375252051 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.138013004Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.138116106Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-3e0023dc-8018-43af-ac86-531375252051","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.140421433Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3e0023dc-8018-43af-ac86-531375252051","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.158686047Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.187102245Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3e0023dc-8018-43af-ac86-531375252051","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.206406131Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3e0023dc-8018-43af-ac86-531375252051","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.206957611Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.207022523Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.207065774Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.207104605Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.220683556Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.227938561Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.465104338Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"uri":"/crucible/0/regions","method":"POST","req_id":"668ed1ee-f2ac-4728-9224-79d9cf694450","remote_addr":"[fd00:1122:3344:101::c]:51333","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.809504677Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"uri":"/crucible/0/regions/3e0023dc-8018-43af-ac86-531375252051","method":"GET","req_id":"18122354-0b91-44ca-a3fb-fdc00bbda727","remote_addr":"[fd00:1122:3344:101::c]:51333","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":107,"response_code":"200"}
{"msg":"path is \"/data/regions/3e0023dc-8018-43af-ac86-531375252051\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.812695071Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"component":"datafile"}
{"msg":"dataset is oxp_d4135165-e105-4373-9d12-4efe0b6e4c50/crucible/regions/3e0023dc-8018-43af-ac86-531375252051","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.822583368Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"component":"datafile"}
{"msg":"region 3e0023dc-8018-43af-ac86-531375252051 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.833311913Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.833462356Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.833548778Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"uri":"/crucible/0/regions/3e0023dc-8018-43af-ac86-531375252051","method":"DELETE","req_id":"24425009-5c9d-458a-9cd4-d5fe29b13755","remote_addr":"[fd00:1122:3344:101::c]:51333","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":20886,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-3e0023dc-8018-43af-ac86-531375252051 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.834595418Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.836422385Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"uri":"/crucible/0/regions/3e0023dc-8018-43af-ac86-531375252051","method":"GET","req_id":"2bf564e0-a705-4c30-903f-f13903745e78","remote_addr":"[fd00:1122:3344:101::c]:51333","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":32,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.851292732Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_d4135165-e105-4373-9d12-4efe0b6e4c50/crucible/regions/3e0023dc-8018-43af-ac86-531375252051\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.86272085Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"region":"3e0023dc-8018-43af-ac86-531375252051","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.90675766Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"uri":"/crucible/0/regions/3e0023dc-8018-43af-ac86-531375252051","method":"GET","req_id":"0fff4c92-45a4-4bd6-be62-b943ad6add6e","remote_addr":"[fd00:1122:3344:101::c]:51333","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":97,"response_code":"200"}
{"msg":"region 3e0023dc-8018-43af-ac86-531375252051 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:47.934547135Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-04T22:14:48.087744145Z","hostname":"oxz_crucible_06c64b28-3935-456b-b9a9-8035576b6e7a","pid":21104,"uri":"/crucible/0/regions/3e0023dc-8018-43af-ac86-531375252051","method":"GET","req_id":"7d4eedad-d763-478a-9375-79f30e367730","remote_addr":"[fd00:1122:3344:101::c]:51333","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":45,"response_code":"200"}