[ Dec  5 07:27:02 Enabled. ]
[ Dec  5 07:27:02 Rereading configuration. ]
[ Dec  5 07:27:04 Rereading configuration. ]
[ Dec  5 07:27:07 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_a907f9b6-2004-431b-b649-347f38cf3630/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_a907f9b6-2004-431b-b649-347f38cf3630/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:07.56389927Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:07.568734892Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:07.568745222Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:07.582578268Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:07.65725693Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:29.351734168Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:33369"}
{"msg":"region 29488fa0-2614-4f36-8cc3-2daa281d9623 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:29.386436601Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:29.391362574Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:29.395803Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"uri":"/crucible/0/regions","method":"POST","req_id":"378b8c85-7535-4560-84d7-4264215a94c4","remote_addr":"[fd00:1122:3344:101::c]:33369","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":11731,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_a907f9b6-2004-431b-b649-347f38cf3630/crucible/regions/29488fa0-2614-4f36-8cc3-2daa281d9623","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:29.402333116Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_a907f9b6-2004-431b-b649-347f38cf3630/crucible/regions/29488fa0-2614-4f36-8cc3-2daa281d9623","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:29.402403117Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"29488fa0-2614-4f36-8cc3-2daa281d9623\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/29488fa0-2614-4f36-8cc3-2daa281d9623\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:29.471410397Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"region":"29488fa0-2614-4f36-8cc3-2daa281d9623","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"29488fa0-2614-4f36-8cc3-2daa281d9623\" \"--data\" \"/data/regions/29488fa0-2614-4f36-8cc3-2daa281d9623\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:29.473746132Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"region":"29488fa0-2614-4f36-8cc3-2daa281d9623","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:29.754792607Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"region":"29488fa0-2614-4f36-8cc3-2daa281d9623","component":"worker"}
{"msg":"region 29488fa0-2614-4f36-8cc3-2daa281d9623 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:29.754882168Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:29.755077011Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-29488fa0-2614-4f36-8cc3-2daa281d9623","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:29.755508677Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-29488fa0-2614-4f36-8cc3-2daa281d9623","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:29.760228837Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:29.771112548Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"uri":"/crucible/0/regions","method":"POST","req_id":"56c595a7-8011-4305-b430-44c10b4c057a","remote_addr":"[fd00:1122:3344:101::c]:33369","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":63,"response_code":"200"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:29.771333331Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-29488fa0-2614-4f36-8cc3-2daa281d9623","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:29.774569589Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/29488fa0-2614-4f36-8cc3-2daa281d9623","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:29.77532682Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:29.775398221Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:29.775441182Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:29.775484982Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:29.779910798Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:29.784533616Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:30.09596209Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"uri":"/crucible/0/regions/29488fa0-2614-4f36-8cc3-2daa281d9623","method":"GET","req_id":"011ade33-1713-4bb8-ba95-0ef6a1f7738e","remote_addr":"[fd00:1122:3344:101::c]:33369","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":71,"response_code":"200"}
{"msg":"path is \"/data/regions/29488fa0-2614-4f36-8cc3-2daa281d9623\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:30.098907923Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"datafile"}
{"msg":"dataset is oxp_a907f9b6-2004-431b-b649-347f38cf3630/crucible/regions/29488fa0-2614-4f36-8cc3-2daa281d9623","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:30.11018105Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"datafile"}
{"msg":"region 29488fa0-2614-4f36-8cc3-2daa281d9623 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:30.120081116Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:30.12029921Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:30.120362191Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"uri":"/crucible/0/regions/29488fa0-2614-4f36-8cc3-2daa281d9623","method":"DELETE","req_id":"6f9d2fba-a175-46c8-9053-e18189d87db5","remote_addr":"[fd00:1122:3344:101::c]:33369","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":21425,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-29488fa0-2614-4f36-8cc3-2daa281d9623 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:30.12372209Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:30.124172617Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"uri":"/crucible/0/regions/29488fa0-2614-4f36-8cc3-2daa281d9623","method":"GET","req_id":"ed46d999-e40e-4770-bcfc-42b45ff07c62","remote_addr":"[fd00:1122:3344:101::c]:33369","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":47,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:30.129924962Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_a907f9b6-2004-431b-b649-347f38cf3630/crucible/regions/29488fa0-2614-4f36-8cc3-2daa281d9623\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:30.14063042Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"region":"29488fa0-2614-4f36-8cc3-2daa281d9623","component":"worker"}
{"msg":"region 29488fa0-2614-4f36-8cc3-2daa281d9623 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:30.216950918Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:27:30.275429613Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"uri":"/crucible/0/regions/29488fa0-2614-4f36-8cc3-2daa281d9623","method":"GET","req_id":"7e189d30-520e-43d4-85aa-fbab3849343b","remote_addr":"[fd00:1122:3344:101::c]:33369","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":74,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:28:20.395153554Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:57101"}
{"msg":"region dc2b87cd-f100-4973-90dd-6cb01aeab9a1 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:28:20.395344237Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:28:20.395501189Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"uri":"/crucible/0/regions","method":"POST","req_id":"f8a802be-9b3c-4814-99b2-e2b81ba84217","remote_addr":"[fd00:1122:3344:101::a]:57101","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":205,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:28:20.395517709Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_a907f9b6-2004-431b-b649-347f38cf3630/crucible/regions/dc2b87cd-f100-4973-90dd-6cb01aeab9a1","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:28:20.404485462Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_a907f9b6-2004-431b-b649-347f38cf3630/crucible/regions/dc2b87cd-f100-4973-90dd-6cb01aeab9a1","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:28:20.404516993Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"dc2b87cd-f100-4973-90dd-6cb01aeab9a1\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/dc2b87cd-f100-4973-90dd-6cb01aeab9a1\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:28:20.448464812Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"region":"dc2b87cd-f100-4973-90dd-6cb01aeab9a1","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"dc2b87cd-f100-4973-90dd-6cb01aeab9a1\" \"--data\" \"/data/regions/dc2b87cd-f100-4973-90dd-6cb01aeab9a1\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:28:20.45034904Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"region":"dc2b87cd-f100-4973-90dd-6cb01aeab9a1","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:28:20.523204607Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"region":"dc2b87cd-f100-4973-90dd-6cb01aeab9a1","component":"worker"}
{"msg":"region dc2b87cd-f100-4973-90dd-6cb01aeab9a1 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:28:20.523230787Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:28:20.523345369Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-dc2b87cd-f100-4973-90dd-6cb01aeab9a1","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:28:20.52475368Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-dc2b87cd-f100-4973-90dd-6cb01aeab9a1","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:28:20.52815463Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:28:20.536332241Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-dc2b87cd-f100-4973-90dd-6cb01aeab9a1","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:28:20.539769192Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/dc2b87cd-f100-4973-90dd-6cb01aeab9a1","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:28:20.540044536Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:28:20.540101147Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:28:20.540142207Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:28:20.540182848Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:28:20.544464021Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:28:20.549191541Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-05T07:28:20.69318466Z","hostname":"oxz_crucible_96a6e8a1-9cef-46c0-8923-6b2827260665","pid":24879,"uri":"/crucible/0/regions","method":"POST","req_id":"cea3eb97-2c37-44ea-830c-264380479eef","remote_addr":"[fd00:1122:3344:101::a]:57101","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":56,"response_code":"200"}