[ Apr  3 10:09:32 Enabled. ]
[ Apr  3 10:09:33 Rereading configuration. ]
[ Apr  3 10:09:36 Rereading configuration. ]
[ Apr  3 10:09:43 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_5df238e3-5630-4184-a188-36117b1a7b7b/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_5df238e3-5630-4184-a188-36117b1a7b7b/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:09:43.621203285Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:09:43.621510261Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:09:43.621519661Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:09:43.771877456Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:32.766577028Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:49999"}
{"msg":"region 141660de-7ef1-4018-be2f-3a3c01faac1a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:32.80284543Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:32.808370301Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:32.808429042Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"uri":"/crucible/0/regions","method":"POST","req_id":"7e73ef3b-d141-4a41-b767-07f711fe1073","remote_addr":"[fd00:1122:3344:101::b]:49999","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":10019,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_5df238e3-5630-4184-a188-36117b1a7b7b/crucible/regions/141660de-7ef1-4018-be2f-3a3c01faac1a","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:32.821846117Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_5df238e3-5630-4184-a188-36117b1a7b7b/crucible/regions/141660de-7ef1-4018-be2f-3a3c01faac1a","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:32.821897748Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"141660de-7ef1-4018-be2f-3a3c01faac1a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/141660de-7ef1-4018-be2f-3a3c01faac1a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:32.8904565Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"region":"141660de-7ef1-4018-be2f-3a3c01faac1a","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.022701545Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"region":"141660de-7ef1-4018-be2f-3a3c01faac1a","component":"worker"}
{"msg":"region 141660de-7ef1-4018-be2f-3a3c01faac1a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.022760386Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.022970239Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-141660de-7ef1-4018-be2f-3a3c01faac1a","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.023387317Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-141660de-7ef1-4018-be2f-3a3c01faac1a","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.029720522Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.041349655Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"uri":"/crucible/0/regions","method":"POST","req_id":"c7dbe111-cff0-4ff9-837e-4ef7ce56526c","remote_addr":"[fd00:1122:3344:101::b]:49999","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.042280542Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-141660de-7ef1-4018-be2f-3a3c01faac1a","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.04652616Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/141660de-7ef1-4018-be2f-3a3c01faac1a","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.047401805Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.047487947Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.047543248Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.047577119Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.052271374Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.057606152Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.594400252Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:45926"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.594477784Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"uri":"/crucible/0/regions/141660de-7ef1-4018-be2f-3a3c01faac1a","method":"GET","req_id":"13ae7c62-fd44-48d9-b348-7d73bd83cf93","remote_addr":"[fd00:1122:3344:101::b]:45926","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"path is \"/data/regions/141660de-7ef1-4018-be2f-3a3c01faac1a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.594664117Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"datafile"}
{"msg":"dataset is oxp_5df238e3-5630-4184-a188-36117b1a7b7b/crucible/regions/141660de-7ef1-4018-be2f-3a3c01faac1a","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.603771414Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"datafile"}
{"msg":"region 141660de-7ef1-4018-be2f-3a3c01faac1a state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.611276091Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.611501065Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.611567656Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"uri":"/crucible/0/regions/141660de-7ef1-4018-be2f-3a3c01faac1a","method":"DELETE","req_id":"fdcd85c9-c339-47ca-a23a-eb660748c871","remote_addr":"[fd00:1122:3344:101::b]:45926","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":16927,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.612277539Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"uri":"/crucible/0/regions/141660de-7ef1-4018-be2f-3a3c01faac1a","method":"GET","req_id":"086f05ff-49ac-4330-8c56-06c982e740c1","remote_addr":"[fd00:1122:3344:101::b]:45926","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":589,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-141660de-7ef1-4018-be2f-3a3c01faac1a (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.612798838Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.616858682Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_5df238e3-5630-4184-a188-36117b1a7b7b/crucible/regions/141660de-7ef1-4018-be2f-3a3c01faac1a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.626495989Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"region":"141660de-7ef1-4018-be2f-3a3c01faac1a","component":"worker"}
{"msg":"region 141660de-7ef1-4018-be2f-3a3c01faac1a state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.687579514Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:33.719610149Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"uri":"/crucible/0/regions/141660de-7ef1-4018-be2f-3a3c01faac1a","method":"GET","req_id":"97032347-cf17-46a8-be9a-59f9b6f0d80f","remote_addr":"[fd00:1122:3344:101::b]:45926","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":68,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:55.400839155Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:52972"}
{"msg":"region eafdd8dd-11e7-485f-9292-848016ea93f0 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:55.430917964Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:55.431302471Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"uri":"/crucible/0/regions","method":"POST","req_id":"8133ed24-259d-4104-a5f2-df22e0d093c6","remote_addr":"[fd00:1122:3344:101::a]:52972","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":431,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:55.431346252Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_5df238e3-5630-4184-a188-36117b1a7b7b/crucible/regions/eafdd8dd-11e7-485f-9292-848016ea93f0","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:55.440078441Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_5df238e3-5630-4184-a188-36117b1a7b7b/crucible/regions/eafdd8dd-11e7-485f-9292-848016ea93f0","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:55.440105701Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"eafdd8dd-11e7-485f-9292-848016ea93f0\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/eafdd8dd-11e7-485f-9292-848016ea93f0\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:55.506900661Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"region":"eafdd8dd-11e7-485f-9292-848016ea93f0","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:55.579997075Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"region":"eafdd8dd-11e7-485f-9292-848016ea93f0","component":"worker"}
{"msg":"region eafdd8dd-11e7-485f-9292-848016ea93f0 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:55.580034156Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:55.58023811Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:55.581660676Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"uri":"/crucible/0/regions","method":"POST","req_id":"a5d2eae0-a444-48c3-89c9-97168c5d799d","remote_addr":"[fd00:1122:3344:101::a]:52972","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":684,"response_code":"200"}
{"msg":"creating missing downstairs instance downstairs-eafdd8dd-11e7-485f-9292-848016ea93f0","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:55.581737017Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-eafdd8dd-11e7-485f-9292-848016ea93f0","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:55.584783483Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:55.592430662Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-eafdd8dd-11e7-485f-9292-848016ea93f0","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:55.595789194Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/eafdd8dd-11e7-485f-9292-848016ea93f0","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:55.596060009Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:55.59610296Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:55.59614848Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:55.596183211Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:55.600422298Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T10:10:55.605316838Z","hostname":"oxz_crucible_ddcb991a-c8a0-472d-a34d-1dcd46abb727","pid":22136,"component":"worker"}