[ Mar 25 22:28:16 Enabled. ]
[ Mar 25 22:28:16 Rereading configuration. ]
[ Mar 25 22:28:18 Rereading configuration. ]
[ Mar 25 22:28:24 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_db2f7830-be20-44d7-ae1f-5a4724d29a8b/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_db2f7830-be20-44d7-ae1f-5a4724d29a8b/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:28:24.460696418Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:28:24.461722927Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:28:24.461730477Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:28:24.482121539Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:28:24.618345083Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:15.610944029Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:35877"}
{"msg":"region 287c9f45-236f-4e18-b60c-028c1c93bb84 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:15.656246025Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:15.658905184Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"uri":"/crucible/0/regions","method":"POST","req_id":"a25e24c6-2fa7-4e29-90e4-7b40c0462a20","remote_addr":"[fd00:1122:3344:101::c]:35877","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":19004,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:15.661353989Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_db2f7830-be20-44d7-ae1f-5a4724d29a8b/crucible/regions/287c9f45-236f-4e18-b60c-028c1c93bb84","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:15.672709506Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_db2f7830-be20-44d7-ae1f-5a4724d29a8b/crucible/regions/287c9f45-236f-4e18-b60c-028c1c93bb84","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:15.672741146Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"287c9f45-236f-4e18-b60c-028c1c93bb84\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/287c9f45-236f-4e18-b60c-028c1c93bb84\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:15.721820241Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"region":"287c9f45-236f-4e18-b60c-028c1c93bb84","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"287c9f45-236f-4e18-b60c-028c1c93bb84\" \"--data\" \"/data/regions/287c9f45-236f-4e18-b60c-028c1c93bb84\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:15.72393751Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"region":"287c9f45-236f-4e18-b60c-028c1c93bb84","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:15.855969318Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"region":"287c9f45-236f-4e18-b60c-028c1c93bb84","component":"worker"}
{"msg":"region 287c9f45-236f-4e18-b60c-028c1c93bb84 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:15.856031909Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:15.856108201Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-287c9f45-236f-4e18-b60c-028c1c93bb84","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:15.856536938Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-287c9f45-236f-4e18-b60c-028c1c93bb84","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:15.863064968Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:15.87363398Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-287c9f45-236f-4e18-b60c-028c1c93bb84","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:15.876424661Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/287c9f45-236f-4e18-b60c-028c1c93bb84","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:15.877162895Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:15.877223646Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:15.877264516Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:15.877302297Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:15.882015103Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:15.891911454Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:15.953505707Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"uri":"/crucible/0/regions","method":"POST","req_id":"c8f52e9b-f73e-4977-81d8-84414018220b","remote_addr":"[fd00:1122:3344:101::c]:35877","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":67,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:16.2623751Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"uri":"/crucible/0/regions/287c9f45-236f-4e18-b60c-028c1c93bb84","method":"GET","req_id":"0df06b20-bdb9-47e1-b0ad-a47475487be5","remote_addr":"[fd00:1122:3344:101::c]:35877","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":2057,"response_code":"200"}
{"msg":"path is \"/data/regions/287c9f45-236f-4e18-b60c-028c1c93bb84\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:16.267133877Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"datafile"}
{"msg":"dataset is oxp_db2f7830-be20-44d7-ae1f-5a4724d29a8b/crucible/regions/287c9f45-236f-4e18-b60c-028c1c93bb84","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:16.278339111Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"datafile"}
{"msg":"region 287c9f45-236f-4e18-b60c-028c1c93bb84 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:16.29142528Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:16.291563212Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:16.291614723Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"uri":"/crucible/0/regions/287c9f45-236f-4e18-b60c-028c1c93bb84","method":"DELETE","req_id":"799142df-d99c-4773-b1ac-8941c518d51c","remote_addr":"[fd00:1122:3344:101::c]:35877","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":24510,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-287c9f45-236f-4e18-b60c-028c1c93bb84 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:16.292636792Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:16.294588178Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"uri":"/crucible/0/regions/287c9f45-236f-4e18-b60c-028c1c93bb84","method":"GET","req_id":"08f407d8-cdfe-4aa8-ba17-6057c9271d00","remote_addr":"[fd00:1122:3344:101::c]:35877","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":30,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:16.296409541Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_db2f7830-be20-44d7-ae1f-5a4724d29a8b/crucible/regions/287c9f45-236f-4e18-b60c-028c1c93bb84\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:16.307410681Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"region":"287c9f45-236f-4e18-b60c-028c1c93bb84","component":"worker"}
{"msg":"region 287c9f45-236f-4e18-b60c-028c1c93bb84 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:16.366806335Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:16.41700462Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"uri":"/crucible/0/regions/287c9f45-236f-4e18-b60c-028c1c93bb84","method":"GET","req_id":"e9536b13-ad82-465e-9aef-73c62846f10a","remote_addr":"[fd00:1122:3344:101::c]:35877","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":51,"response_code":"200"}
{"msg":"region e7274c9e-8434-4084-8eea-7823f3327499 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:18.116010547Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:18.11615914Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:18.11616857Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"uri":"/crucible/0/regions","method":"POST","req_id":"9a8c24a1-0e76-4381-bbcc-9792e6ddc120","remote_addr":"[fd00:1122:3344:101::c]:35877","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":218,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_db2f7830-be20-44d7-ae1f-5a4724d29a8b/crucible/regions/e7274c9e-8434-4084-8eea-7823f3327499","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:18.125553261Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_db2f7830-be20-44d7-ae1f-5a4724d29a8b/crucible/regions/e7274c9e-8434-4084-8eea-7823f3327499","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:18.125596252Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e7274c9e-8434-4084-8eea-7823f3327499\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e7274c9e-8434-4084-8eea-7823f3327499\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:18.172015518Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"region":"e7274c9e-8434-4084-8eea-7823f3327499","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e7274c9e-8434-4084-8eea-7823f3327499\" \"--data\" \"/data/regions/e7274c9e-8434-4084-8eea-7823f3327499\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:18.173991724Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"region":"e7274c9e-8434-4084-8eea-7823f3327499","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:18.247690298Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"region":"e7274c9e-8434-4084-8eea-7823f3327499","component":"worker"}
{"msg":"region e7274c9e-8434-4084-8eea-7823f3327499 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:18.247726259Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:18.247846061Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e7274c9e-8434-4084-8eea-7823f3327499","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:18.249526002Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e7274c9e-8434-4084-8eea-7823f3327499","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:18.252491436Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:18.260143575Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e7274c9e-8434-4084-8eea-7823f3327499","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:18.263287623Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e7274c9e-8434-4084-8eea-7823f3327499","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:18.263547598Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:18.263607089Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:18.26365343Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:18.26369739Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:18.267142423Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:18.271640245Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T22:29:18.385909429Z","hostname":"oxz_crucible_70ea4d72-7e14-4633-89ed-caec6fc03ffa","pid":21983,"uri":"/crucible/0/regions","method":"POST","req_id":"eeb86bea-42b6-4093-a118-254af41e1681","remote_addr":"[fd00:1122:3344:101::c]:35877","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":65,"response_code":"200"}