[ May 29 21:15:56 Enabled. ]
[ May 29 21:15:56 Rereading configuration. ]
[ May 29 21:15:59 Rereading configuration. ]
[ May 29 21:16:03 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_ebc4bed0-dc72-4f62-8305-1ac1d8a2fab1/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_ebc4bed0-dc72-4f62-8305-1ac1d8a2fab1/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:16:03.159344766Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:16:03.162781802Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:16:03.162808903Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:16:03.179444085Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:16:03.319677835Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.322790263Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63227"}
{"msg":"region c51e8003-95d4-4e48-800b-8e82254e79e4 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.389696294Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.39699237Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"uri":"/crucible/0/regions","method":"POST","req_id":"50e50bab-10c1-46c3-876d-08e91a923f30","remote_addr":"[fd00:1122:3344:101::a]:63227","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":16136,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.397014451Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_ebc4bed0-dc72-4f62-8305-1ac1d8a2fab1/crucible/regions/c51e8003-95d4-4e48-800b-8e82254e79e4","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.409022897Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ebc4bed0-dc72-4f62-8305-1ac1d8a2fab1/crucible/regions/c51e8003-95d4-4e48-800b-8e82254e79e4","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.409053968Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"c51e8003-95d4-4e48-800b-8e82254e79e4\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c51e8003-95d4-4e48-800b-8e82254e79e4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.459087579Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"region":"c51e8003-95d4-4e48-800b-8e82254e79e4","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c51e8003-95d4-4e48-800b-8e82254e79e4\" \"--data\" \"/data/regions/c51e8003-95d4-4e48-800b-8e82254e79e4\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.461401487Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"region":"c51e8003-95d4-4e48-800b-8e82254e79e4","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.653985886Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"uri":"/crucible/0/regions","method":"POST","req_id":"6531f660-9815-4347-8674-45a9109d5a69","remote_addr":"[fd00:1122:3344:101::a]:63227","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.756003323Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"region":"c51e8003-95d4-4e48-800b-8e82254e79e4","component":"worker"}
{"msg":"region c51e8003-95d4-4e48-800b-8e82254e79e4 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.756046245Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.756149778Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-c51e8003-95d4-4e48-800b-8e82254e79e4","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.756570893Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c51e8003-95d4-4e48-800b-8e82254e79e4","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.763582199Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.775155061Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c51e8003-95d4-4e48-800b-8e82254e79e4","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.780013975Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c51e8003-95d4-4e48-800b-8e82254e79e4","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.786144212Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.78814204Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.788227643Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.788289334Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.793125918Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.797686612Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:17:15.926367891Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"uri":"/crucible/0/regions","method":"POST","req_id":"2bbc224f-cd94-4fcf-b0ac-4e6e83a378ae","remote_addr":"[fd00:1122:3344:101::a]:63227","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.645775008Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:40693"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.648632145Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"uri":"/crucible/0/regions/c51e8003-95d4-4e48-800b-8e82254e79e4","method":"GET","req_id":"5e550021-6dd9-4a83-81bc-5d3aed12eb2d","remote_addr":"[fd00:1122:3344:101::a]:40693","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":2827,"response_code":"200"}
{"msg":"path is \"/data/regions/c51e8003-95d4-4e48-800b-8e82254e79e4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.648828691Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"datafile"}
{"msg":"dataset is oxp_ebc4bed0-dc72-4f62-8305-1ac1d8a2fab1/crucible/regions/c51e8003-95d4-4e48-800b-8e82254e79e4","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.657848716Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"datafile"}
{"msg":"snapshot is oxp_ebc4bed0-dc72-4f62-8305-1ac1d8a2fab1/crucible/regions/c51e8003-95d4-4e48-800b-8e82254e79e4@dc5d364c-5dc8-4513-a3ee-bba5cdbfec48","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.666444157Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ebc4bed0-dc72-4f62-8305-1ac1d8a2fab1/crucible/regions/c51e8003-95d4-4e48-800b-8e82254e79e4\", \"dc5d364c-5dc8-4513-a3ee-bba5cdbfec48\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.666477168Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.677729868Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"uri":"/crucible/0/regions/c51e8003-95d4-4e48-800b-8e82254e79e4/snapshots/dc5d364c-5dc8-4513-a3ee-bba5cdbfec48","method":"GET","req_id":"727dba3b-e241-4ceb-9cfa-83674d44188c","remote_addr":"[fd00:1122:3344:101::a]:40693","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":28928,"response_code":"200"}
{"msg":"path is \"/data/regions/c51e8003-95d4-4e48-800b-8e82254e79e4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.677951146Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"datafile"}
{"msg":"dataset is oxp_ebc4bed0-dc72-4f62-8305-1ac1d8a2fab1/crucible/regions/c51e8003-95d4-4e48-800b-8e82254e79e4","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.686816855Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"datafile"}
{"msg":"snapshot is oxp_ebc4bed0-dc72-4f62-8305-1ac1d8a2fab1/crucible/regions/c51e8003-95d4-4e48-800b-8e82254e79e4@dc5d364c-5dc8-4513-a3ee-bba5cdbfec48","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.694565017Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ebc4bed0-dc72-4f62-8305-1ac1d8a2fab1/crucible/regions/c51e8003-95d4-4e48-800b-8e82254e79e4\", \"dc5d364c-5dc8-4513-a3ee-bba5cdbfec48\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.694622699Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot c51e8003-95d4-4e48-800b-8e82254e79e4-dc5d364c-5dc8-4513-a3ee-bba5cdbfec48 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.702473204Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"datafile"}
{"msg":"applying SMF actions for region c51e8003-95d4-4e48-800b-8e82254e79e4 running snapshot dc5d364c-5dc8-4513-a3ee-bba5cdbfec48 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.703070784Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.703085765Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"uri":"/crucible/0/regions/c51e8003-95d4-4e48-800b-8e82254e79e4/snapshots/dc5d364c-5dc8-4513-a3ee-bba5cdbfec48/run","method":"POST","req_id":"70d59a49-527e-4fce-9dff-cf3059458f10","remote_addr":"[fd00:1122:3344:101::a]:40693","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":25168,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/c51e8003-95d4-4e48-800b-8e82254e79e4","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.708157147Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.708241559Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.708326952Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.708334942Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.708379054Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c51e8003-95d4-4e48-800b-8e82254e79e4","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.708430016Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c51e8003-95d4-4e48-800b-8e82254e79e4","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.708528459Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.708591551Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.708639373Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.708696605Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.715142392Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-c51e8003-95d4-4e48-800b-8e82254e79e4-dc5d364c-5dc8-4513-a3ee-bba5cdbfec48","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.715603478Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-c51e8003-95d4-4e48-800b-8e82254e79e4-dc5d364c-5dc8-4513-a3ee-bba5cdbfec48","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.718252738Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-c51e8003-95d4-4e48-800b-8e82254e79e4-dc5d364c-5dc8-4513-a3ee-bba5cdbfec48","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.730423689Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c51e8003-95d4-4e48-800b-8e82254e79e4/.zfs/snapshot/dc5d364c-5dc8-4513-a3ee-bba5cdbfec48","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.730651597Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.730702998Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.73074348Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.730780651Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.730826033Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.733847955Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.738055987Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"region c51e8003-95d4-4e48-800b-8e82254e79e4 running snapshot dc5d364c-5dc8-4513-a3ee-bba5cdbfec48 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:22.738069717Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.61961618Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:54248"}
{"msg":"region 1da8395b-e899-4444-bed7-f52d66a91749 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.679154232Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.679327338Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"uri":"/crucible/0/regions","method":"POST","req_id":"2b6400ed-d1eb-4917-9b36-d46f8116e4ac","remote_addr":"[fd00:1122:3344:101::a]:54248","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":231,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.679355708Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_ebc4bed0-dc72-4f62-8305-1ac1d8a2fab1/crucible/regions/1da8395b-e899-4444-bed7-f52d66a91749","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.689379677Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ebc4bed0-dc72-4f62-8305-1ac1d8a2fab1/crucible/regions/1da8395b-e899-4444-bed7-f52d66a91749","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.689411698Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"1da8395b-e899-4444-bed7-f52d66a91749\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/1da8395b-e899-4444-bed7-f52d66a91749\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.745951589Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"region":"1da8395b-e899-4444-bed7-f52d66a91749","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"1da8395b-e899-4444-bed7-f52d66a91749\" \"--data\" \"/data/regions/1da8395b-e899-4444-bed7-f52d66a91749\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.747789211Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"region":"1da8395b-e899-4444-bed7-f52d66a91749","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.831307524Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"region":"1da8395b-e899-4444-bed7-f52d66a91749","component":"worker"}
{"msg":"region 1da8395b-e899-4444-bed7-f52d66a91749 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.831350355Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.83148239Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-1da8395b-e899-4444-bed7-f52d66a91749","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.831970336Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-1da8395b-e899-4444-bed7-f52d66a91749","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.838197667Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.845178393Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1da8395b-e899-4444-bed7-f52d66a91749","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.847867763Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1da8395b-e899-4444-bed7-f52d66a91749","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.848142943Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.848196345Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.848236896Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.848274627Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.851650181Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/c51e8003-95d4-4e48-800b-8e82254e79e4","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.856076371Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.856196285Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.856281788Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.856290158Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.85633438Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c51e8003-95d4-4e48-800b-8e82254e79e4","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.856381881Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c51e8003-95d4-4e48-800b-8e82254e79e4","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.860466589Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.860523001Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.860572633Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.860620385Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.86402672Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.86581467Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:25.882158793Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"uri":"/crucible/0/regions","method":"POST","req_id":"7c55e409-b15e-4d19-b0dc-550d97fb5b63","remote_addr":"[fd00:1122:3344:101::a]:54248","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":71,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:48.844825885Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:59542"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:48.844928498Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"uri":"/crucible/0/regions/1da8395b-e899-4444-bed7-f52d66a91749","method":"GET","req_id":"7b64ac38-97ba-4e26-91f3-76f960680afc","remote_addr":"[fd00:1122:3344:101::a]:59542","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":74,"response_code":"200"}
{"msg":"path is \"/data/regions/1da8395b-e899-4444-bed7-f52d66a91749\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:48.845131005Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"datafile"}
{"msg":"dataset is oxp_ebc4bed0-dc72-4f62-8305-1ac1d8a2fab1/crucible/regions/1da8395b-e899-4444-bed7-f52d66a91749","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:48.856589172Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"datafile"}
{"msg":"region 1da8395b-e899-4444-bed7-f52d66a91749 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:48.864705626Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:48.864913123Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:48.867403417Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"uri":"/crucible/0/regions/1da8395b-e899-4444-bed7-f52d66a91749","method":"DELETE","req_id":"dca960b6-65ad-4b48-aec3-4d1093a50fd4","remote_addr":"[fd00:1122:3344:101::a]:59542","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":22320,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:48.867557303Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"uri":"/crucible/0/regions/1da8395b-e899-4444-bed7-f52d66a91749","method":"GET","req_id":"96e7969e-eae2-4ce9-bb97-7292c09be847","remote_addr":"[fd00:1122:3344:101::a]:59542","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":29,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-1da8395b-e899-4444-bed7-f52d66a91749 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:48.869209569Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/c51e8003-95d4-4e48-800b-8e82254e79e4","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:48.873656089Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:48.873743322Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:48.873826085Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:48.873836255Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:48.873874976Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c51e8003-95d4-4e48-800b-8e82254e79e4","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:48.873922268Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c51e8003-95d4-4e48-800b-8e82254e79e4","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:48.87426999Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:48.874348142Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:48.874408504Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:48.874455096Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:48.878210073Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:48.879653811Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_ebc4bed0-dc72-4f62-8305-1ac1d8a2fab1/crucible/regions/1da8395b-e899-4444-bed7-f52d66a91749\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:48.889113701Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"region":"1da8395b-e899-4444-bed7-f52d66a91749","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:48.928359878Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"uri":"/crucible/0/regions/1da8395b-e899-4444-bed7-f52d66a91749","method":"GET","req_id":"f354cf79-2fac-42a1-9a0e-1ca3f8a3c1b5","remote_addr":"[fd00:1122:3344:101::a]:59542","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":567,"response_code":"200"}
{"msg":"region 1da8395b-e899-4444-bed7-f52d66a91749 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:48.936191192Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:22:49.099570644Z","hostname":"oxz_crucible_1c81c746-a1ff-4e95-bd30-44d2b2f90415","pid":26461,"uri":"/crucible/0/regions/1da8395b-e899-4444-bed7-f52d66a91749","method":"GET","req_id":"7f015402-a89b-49cf-bc8e-28119b484df3","remote_addr":"[fd00:1122:3344:101::a]:59542","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":41,"response_code":"200"}