[ Feb 28 13:20:41 Enabled. ]
[ Feb 28 13:20:41 Rereading configuration. ]
[ Feb 28 13:20:43 Rereading configuration. ]
[ Feb 28 13:20:46 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:20:46.898075058Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:20:46.899552685Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:20:46.899561626Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:20:47.063904542Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:22:08.392157404Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:37102"}
{"msg":"region e21e29f0-a653-4424-b3c9-9c42c31fe20b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:22:08.453692741Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:22:08.453949917Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"uri":"/crucible/0/regions","method":"POST","req_id":"2560f457-b18f-4ac8-82d5-bc2be005b1a9","remote_addr":"[fd00:1122:3344:101::a]:37102","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":1603,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:22:08.454584714Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/e21e29f0-a653-4424-b3c9-9c42c31fe20b","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:22:08.464151051Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/e21e29f0-a653-4424-b3c9-9c42c31fe20b","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:22:08.464187912Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e21e29f0-a653-4424-b3c9-9c42c31fe20b\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/e21e29f0-a653-4424-b3c9-9c42c31fe20b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:22:08.517085795Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"region":"e21e29f0-a653-4424-b3c9-9c42c31fe20b","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:22:08.68601662Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"uri":"/crucible/0/regions","method":"POST","req_id":"c0feef56-e1c8-48cb-b2b9-bacae5e4dba9","remote_addr":"[fd00:1122:3344:101::a]:37102","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":83,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:22:08.699238091Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"region":"e21e29f0-a653-4424-b3c9-9c42c31fe20b","component":"worker"}
{"msg":"region e21e29f0-a653-4424-b3c9-9c42c31fe20b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:22:08.699259652Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:22:08.699419856Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e21e29f0-a653-4424-b3c9-9c42c31fe20b","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:22:08.699924399Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e21e29f0-a653-4424-b3c9-9c42c31fe20b","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:22:08.706667933Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:22:08.728033023Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e21e29f0-a653-4424-b3c9-9c42c31fe20b","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:22:08.734426858Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e21e29f0-a653-4424-b3c9-9c42c31fe20b","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:22:08.735319971Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:22:08.735487236Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:22:08.735545647Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:22:08.735587478Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:22:08.740450544Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:22:08.747089355Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:22:09.143287699Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"uri":"/crucible/0/regions","method":"POST","req_id":"194030da-dfbd-4c2a-9a3c-dbfacd26c856","remote_addr":"[fd00:1122:3344:101::a]:37102","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.726924902Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:54522"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.730657799Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"uri":"/crucible/0/regions/e21e29f0-a653-4424-b3c9-9c42c31fe20b","method":"GET","req_id":"63fab5f0-3b9d-4abc-83cf-2343b6465bea","remote_addr":"[fd00:1122:3344:101::a]:54522","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":3682,"response_code":"200"}
{"msg":"path is \"/data/regions/e21e29f0-a653-4424-b3c9-9c42c31fe20b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.731017168Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"datafile"}
{"msg":"dataset is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/e21e29f0-a653-4424-b3c9-9c42c31fe20b","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.74465319Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"datafile"}
{"msg":"snapshot is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/e21e29f0-a653-4424-b3c9-9c42c31fe20b@3400d481-4e0c-4b23-99d9-4ea4d578de90","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.757694356Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/e21e29f0-a653-4424-b3c9-9c42c31fe20b\", \"3400d481-4e0c-4b23-99d9-4ea4d578de90\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.757751647Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.774484968Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"uri":"/crucible/0/regions/e21e29f0-a653-4424-b3c9-9c42c31fe20b/snapshots/3400d481-4e0c-4b23-99d9-4ea4d578de90","method":"GET","req_id":"ea565449-c14b-464b-a06d-28669e7b6f82","remote_addr":"[fd00:1122:3344:101::a]:54522","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":43504,"response_code":"200"}
{"msg":"path is \"/data/regions/e21e29f0-a653-4424-b3c9-9c42c31fe20b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.778828411Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"datafile"}
{"msg":"dataset is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/e21e29f0-a653-4424-b3c9-9c42c31fe20b","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.790487991Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"datafile"}
{"msg":"snapshot is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/e21e29f0-a653-4424-b3c9-9c42c31fe20b@3400d481-4e0c-4b23-99d9-4ea4d578de90","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.803031255Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/e21e29f0-a653-4424-b3c9-9c42c31fe20b\", \"3400d481-4e0c-4b23-99d9-4ea4d578de90\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.803063445Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot e21e29f0-a653-4424-b3c9-9c42c31fe20b-3400d481-4e0c-4b23-99d9-4ea4d578de90 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.815863635Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"datafile"}
{"msg":"applying SMF actions for region e21e29f0-a653-4424-b3c9-9c42c31fe20b running snapshot 3400d481-4e0c-4b23-99d9-4ea4d578de90 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.816773869Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.81680137Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"uri":"/crucible/0/regions/e21e29f0-a653-4424-b3c9-9c42c31fe20b/snapshots/3400d481-4e0c-4b23-99d9-4ea4d578de90/run","method":"POST","req_id":"27d6d7fc-3eb4-4f27-acd7-590b388f2ae3","remote_addr":"[fd00:1122:3344:101::a]:54522","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":39228,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-e21e29f0-a653-4424-b3c9-9c42c31fe20b-3400d481-4e0c-4b23-99d9-4ea4d578de90","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.824814676Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-e21e29f0-a653-4424-b3c9-9c42c31fe20b-3400d481-4e0c-4b23-99d9-4ea4d578de90","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.829822505Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-e21e29f0-a653-4424-b3c9-9c42c31fe20b-3400d481-4e0c-4b23-99d9-4ea4d578de90","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.84397607Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e21e29f0-a653-4424-b3c9-9c42c31fe20b/.zfs/snapshot/3400d481-4e0c-4b23-99d9-4ea4d578de90","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.844431742Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.844505264Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.844571985Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.844634157Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.844719239Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.849549794Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.856175434Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"worker"}
{"msg":"region e21e29f0-a653-4424-b3c9-9c42c31fe20b running snapshot 3400d481-4e0c-4b23-99d9-4ea4d578de90 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:34.856192385Z","hostname":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","pid":24222,"component":"datafile"}