[ Oct  9 04:56:25 Enabled. ]
[ Oct  9 04:56:25 Rereading configuration. ]
[ Oct  9 04:56:30 Rereading configuration. ]
[ Oct  9 04:56:35 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_6fa6258a-7497-4e15-a529-59061c358fa5/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_6fa6258a-7497-4e15-a529-59061c358fa5/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:35.107433226Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:35.109059859Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:35.109069039Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:35.12721544Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:35.291484024Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:57.342960849Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63293"}
{"msg":"region 77ac7736-524a-4386-bca7-7cf4726b1d26 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:57.359001672Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:57.363361214Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"uri":"/crucible/0/regions","method":"POST","req_id":"fb8f5161-833e-40fc-af13-7a4d39d0e566","remote_addr":"[fd00:1122:3344:101::a]:63293","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":6722,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:57.364512011Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_6fa6258a-7497-4e15-a529-59061c358fa5/crucible/regions/77ac7736-524a-4386-bca7-7cf4726b1d26","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:57.374195377Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_6fa6258a-7497-4e15-a529-59061c358fa5/crucible/regions/77ac7736-524a-4386-bca7-7cf4726b1d26","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:57.374249159Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"77ac7736-524a-4386-bca7-7cf4726b1d26\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/77ac7736-524a-4386-bca7-7cf4726b1d26\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:57.441379087Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"region":"77ac7736-524a-4386-bca7-7cf4726b1d26","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"77ac7736-524a-4386-bca7-7cf4726b1d26\" \"--data\" \"/data/regions/77ac7736-524a-4386-bca7-7cf4726b1d26\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:57.445278244Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"region":"77ac7736-524a-4386-bca7-7cf4726b1d26","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:57.561925276Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"region":"77ac7736-524a-4386-bca7-7cf4726b1d26","component":"worker"}
{"msg":"region 77ac7736-524a-4386-bca7-7cf4726b1d26 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:57.562001658Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:57.562214445Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-77ac7736-524a-4386-bca7-7cf4726b1d26","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:57.562635619Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-77ac7736-524a-4386-bca7-7cf4726b1d26","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:57.566628129Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:57.57893411Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-77ac7736-524a-4386-bca7-7cf4726b1d26","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:57.583629403Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/77ac7736-524a-4386-bca7-7cf4726b1d26","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:57.584406798Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:57.58446849Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:57.584506381Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:57.584548803Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:57.590430165Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:57.597763264Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:57.597993961Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"uri":"/crucible/0/regions","method":"POST","req_id":"fccc030a-cf36-4a8e-88c1-e0603a8b43e3","remote_addr":"[fd00:1122:3344:101::a]:63293","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":68,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:58.136052887Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"uri":"/crucible/0/regions/77ac7736-524a-4386-bca7-7cf4726b1d26","method":"GET","req_id":"8b5fe0f7-fc4c-4e29-861a-39b04368a9e7","remote_addr":"[fd00:1122:3344:101::a]:63293","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"path is \"/data/regions/77ac7736-524a-4386-bca7-7cf4726b1d26\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:58.139865471Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"datafile"}
{"msg":"dataset is oxp_6fa6258a-7497-4e15-a529-59061c358fa5/crucible/regions/77ac7736-524a-4386-bca7-7cf4726b1d26","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:58.151891813Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"datafile"}
{"msg":"region 77ac7736-524a-4386-bca7-7cf4726b1d26 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:58.161479605Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:58.161688152Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:58.161760914Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"uri":"/crucible/0/regions/77ac7736-524a-4386-bca7-7cf4726b1d26","method":"DELETE","req_id":"20e4fb50-dc0c-4eba-a440-12870d59be3b","remote_addr":"[fd00:1122:3344:101::a]:63293","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":22717,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-77ac7736-524a-4386-bca7-7cf4726b1d26 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:58.162745557Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:58.165666082Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"uri":"/crucible/0/regions/77ac7736-524a-4386-bca7-7cf4726b1d26","method":"GET","req_id":"c03a39b7-0c9a-4f04-9d91-bb843de91c10","remote_addr":"[fd00:1122:3344:101::a]:63293","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":45,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:58.168653879Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_6fa6258a-7497-4e15-a529-59061c358fa5/crucible/regions/77ac7736-524a-4386-bca7-7cf4726b1d26\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:58.179841124Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"region":"77ac7736-524a-4386-bca7-7cf4726b1d26","component":"worker"}
{"msg":"region 77ac7736-524a-4386-bca7-7cf4726b1d26 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:58.255408327Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:56:58.268826284Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"uri":"/crucible/0/regions/77ac7736-524a-4386-bca7-7cf4726b1d26","method":"GET","req_id":"e151f03e-7911-4d68-bcae-3a4f4670ec7e","remote_addr":"[fd00:1122:3344:101::a]:63293","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":720,"response_code":"200"}
{"msg":"region 260c9d1e-2b9e-4249-b643-d29cd2205b54 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:57:00.129738364Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:57:00.1299258Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"uri":"/crucible/0/regions","method":"POST","req_id":"9a43cac9-4579-45b8-8c55-56664dc0c57c","remote_addr":"[fd00:1122:3344:101::a]:63293","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":259,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:57:00.12994242Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_6fa6258a-7497-4e15-a529-59061c358fa5/crucible/regions/260c9d1e-2b9e-4249-b643-d29cd2205b54","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:57:00.139071248Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_6fa6258a-7497-4e15-a529-59061c358fa5/crucible/regions/260c9d1e-2b9e-4249-b643-d29cd2205b54","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:57:00.13913135Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"260c9d1e-2b9e-4249-b643-d29cd2205b54\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/260c9d1e-2b9e-4249-b643-d29cd2205b54\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:57:00.204030765Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"region":"260c9d1e-2b9e-4249-b643-d29cd2205b54","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"260c9d1e-2b9e-4249-b643-d29cd2205b54\" \"--data\" \"/data/regions/260c9d1e-2b9e-4249-b643-d29cd2205b54\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:57:00.205867105Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"region":"260c9d1e-2b9e-4249-b643-d29cd2205b54","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:57:00.279457353Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"region":"260c9d1e-2b9e-4249-b643-d29cd2205b54","component":"worker"}
{"msg":"region 260c9d1e-2b9e-4249-b643-d29cd2205b54 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:57:00.279511545Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:57:00.279722172Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-260c9d1e-2b9e-4249-b643-d29cd2205b54","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:57:00.281067366Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-260c9d1e-2b9e-4249-b643-d29cd2205b54","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:57:00.28457504Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:57:00.314172935Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:57:00.319781127Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"uri":"/crucible/0/regions","method":"POST","req_id":"0b99ac6d-08a3-46b9-8591-e6e4747ff8f7","remote_addr":"[fd00:1122:3344:101::a]:63293","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":71,"response_code":"200"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-260c9d1e-2b9e-4249-b643-d29cd2205b54","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:57:00.332066218Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/260c9d1e-2b9e-4249-b643-d29cd2205b54","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:57:00.332369478Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:57:00.332418289Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:57:00.332461821Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:57:00.332505712Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:57:00.338938852Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T04:57:00.34409873Z","hostname":"oxz_crucible_ac0be696-3be8-4885-8a46-a26a104cc6db","pid":22509,"component":"worker"}