[ Jul 10 17:05:51 Enabled. ]
[ Jul 10 17:05:51 Rereading configuration. ]
[ Jul 10 17:05:52 Rereading configuration. ]
[ Jul 10 17:05:56 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_448c4814-3ffe-4904-ab9b-37ac9252ea25/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_448c4814-3ffe-4904-ab9b-37ac9252ea25/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:05:56.160907946Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:05:56.164130768Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:05:56.164142248Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:05:56.183570884Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:05:56.292107334Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:21.697392216Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63181"}
{"msg":"region 61f10285-9b41-45be-87c1-93bf49bc8893 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:21.733452893Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:21.737753837Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"uri":"/crucible/0/regions","method":"POST","req_id":"a9456bda-a403-4594-81ad-6bb4049aed32","remote_addr":"[fd00:1122:3344:101::a]:63181","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":20590,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:21.738415599Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_448c4814-3ffe-4904-ab9b-37ac9252ea25/crucible/regions/61f10285-9b41-45be-87c1-93bf49bc8893","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:21.7472491Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_448c4814-3ffe-4904-ab9b-37ac9252ea25/crucible/regions/61f10285-9b41-45be-87c1-93bf49bc8893","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:21.747312891Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"61f10285-9b41-45be-87c1-93bf49bc8893\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/61f10285-9b41-45be-87c1-93bf49bc8893\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:21.833182982Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"region":"61f10285-9b41-45be-87c1-93bf49bc8893","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"61f10285-9b41-45be-87c1-93bf49bc8893\" \"--data\" \"/data/regions/61f10285-9b41-45be-87c1-93bf49bc8893\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:21.835302593Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"region":"61f10285-9b41-45be-87c1-93bf49bc8893","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:22.001318924Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"region":"61f10285-9b41-45be-87c1-93bf49bc8893","component":"worker"}
{"msg":"region 61f10285-9b41-45be-87c1-93bf49bc8893 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:22.001478807Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:22.001715242Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-61f10285-9b41-45be-87c1-93bf49bc8893","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:22.002103199Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-61f10285-9b41-45be-87c1-93bf49bc8893","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:22.005840602Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:22.026853638Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-61f10285-9b41-45be-87c1-93bf49bc8893","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:22.034364963Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/61f10285-9b41-45be-87c1-93bf49bc8893","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:22.035318982Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:22.035393503Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:22.035442444Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:22.035491745Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:22.040628335Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:22.055201786Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:22.072127684Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"uri":"/crucible/0/regions","method":"POST","req_id":"1f904b40-435c-4861-a63a-c28752323725","remote_addr":"[fd00:1122:3344:101::a]:63181","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":56,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:23.099701551Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"uri":"/crucible/0/regions/61f10285-9b41-45be-87c1-93bf49bc8893","method":"GET","req_id":"d3a95a07-3e11-4a92-9880-c3757fe85d18","remote_addr":"[fd00:1122:3344:101::a]:63181","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":96,"response_code":"200"}
{"msg":"path is \"/data/regions/61f10285-9b41-45be-87c1-93bf49bc8893\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:23.104263279Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"datafile"}
{"msg":"dataset is oxp_448c4814-3ffe-4904-ab9b-37ac9252ea25/crucible/regions/61f10285-9b41-45be-87c1-93bf49bc8893","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:23.116527437Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"datafile"}
{"msg":"region 61f10285-9b41-45be-87c1-93bf49bc8893 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:23.127868356Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:23.128094671Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:23.128166742Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"uri":"/crucible/0/regions/61f10285-9b41-45be-87c1-93bf49bc8893","method":"DELETE","req_id":"3026fb99-e2bd-40ff-a8dd-3eab35a020bd","remote_addr":"[fd00:1122:3344:101::a]:63181","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":23935,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-61f10285-9b41-45be-87c1-93bf49bc8893 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:23.129214992Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:23.130908305Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"uri":"/crucible/0/regions/61f10285-9b41-45be-87c1-93bf49bc8893","method":"GET","req_id":"de39073a-7df2-45e5-b5cf-3c128362e640","remote_addr":"[fd00:1122:3344:101::a]:63181","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":29,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:23.136037744Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_448c4814-3ffe-4904-ab9b-37ac9252ea25/crucible/regions/61f10285-9b41-45be-87c1-93bf49bc8893\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:23.150221508Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"region":"61f10285-9b41-45be-87c1-93bf49bc8893","component":"worker"}
{"msg":"region 61f10285-9b41-45be-87c1-93bf49bc8893 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:23.243904241Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:23.280510229Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"uri":"/crucible/0/regions/61f10285-9b41-45be-87c1-93bf49bc8893","method":"GET","req_id":"3f9b7964-597d-4309-8eaf-97432831c34d","remote_addr":"[fd00:1122:3344:101::a]:63181","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"region 599fee72-3f4f-46b2-bb20-a1f7aee2d709 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:25.436258759Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:25.436458363Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:25.436467223Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"uri":"/crucible/0/regions","method":"POST","req_id":"8580b900-f5c5-4391-a975-236441e385d3","remote_addr":"[fd00:1122:3344:101::a]:63181","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":270,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_448c4814-3ffe-4904-ab9b-37ac9252ea25/crucible/regions/599fee72-3f4f-46b2-bb20-a1f7aee2d709","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:25.446523978Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_448c4814-3ffe-4904-ab9b-37ac9252ea25/crucible/regions/599fee72-3f4f-46b2-bb20-a1f7aee2d709","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:25.446584448Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"599fee72-3f4f-46b2-bb20-a1f7aee2d709\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/599fee72-3f4f-46b2-bb20-a1f7aee2d709\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:25.540531116Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"region":"599fee72-3f4f-46b2-bb20-a1f7aee2d709","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"599fee72-3f4f-46b2-bb20-a1f7aee2d709\" \"--data\" \"/data/regions/599fee72-3f4f-46b2-bb20-a1f7aee2d709\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:25.542726788Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"region":"599fee72-3f4f-46b2-bb20-a1f7aee2d709","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:25.594050341Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"uri":"/crucible/0/regions","method":"POST","req_id":"e78b3367-368a-45e8-98a8-5bd2d3f7a343","remote_addr":"[fd00:1122:3344:101::a]:63181","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:25.629718111Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"region":"599fee72-3f4f-46b2-bb20-a1f7aee2d709","component":"worker"}
{"msg":"region 599fee72-3f4f-46b2-bb20-a1f7aee2d709 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:25.629798163Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:25.630077978Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-599fee72-3f4f-46b2-bb20-a1f7aee2d709","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:25.631467425Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-599fee72-3f4f-46b2-bb20-a1f7aee2d709","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:25.634227618Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:25.644919615Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-599fee72-3f4f-46b2-bb20-a1f7aee2d709","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:25.649669887Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/599fee72-3f4f-46b2-bb20-a1f7aee2d709","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:25.649966063Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:25.650020514Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:25.650075535Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:25.650091245Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:25.654220375Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:25.658942207Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-10T17:06:25.893390441Z","hostname":"oxz_crucible_49c85468-dfb8-4829-a913-1b1edef71c1b","pid":24047,"uri":"/crucible/0/regions","method":"POST","req_id":"7c0d19c5-57da-4f3d-8ee8-7efb0cc8f0ed","remote_addr":"[fd00:1122:3344:101::a]:63181","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":49,"response_code":"200"}