[ Jul 18 22:03:05 Enabled. ]
[ Jul 18 22:03:05 Rereading configuration. ]
[ Jul 18 22:03:07 Rereading configuration. ]
[ Jul 18 22:03:09 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_18924fa5-0339-4445-8677-913f8ef37483/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_18924fa5-0339-4445-8677-913f8ef37483/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:09.3549964Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:09.356629643Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:09.356644143Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:09.379209324Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:09.470054997Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:04:25.435914313Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:54187"}
{"msg":"region 6cdf6550-5363-4a93-9776-188667afbbd5 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:04:25.454518998Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:04:25.461360036Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"uri":"/crucible/0/regions","method":"POST","req_id":"2e1869f4-015b-42c1-9cf7-9794dcdd50de","remote_addr":"[fd00:1122:3344:101::a]:54187","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":14870,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:04:25.461379246Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_18924fa5-0339-4445-8677-913f8ef37483/crucible/regions/6cdf6550-5363-4a93-9776-188667afbbd5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:04:25.474405481Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_18924fa5-0339-4445-8677-913f8ef37483/crucible/regions/6cdf6550-5363-4a93-9776-188667afbbd5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:04:25.474440232Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"6cdf6550-5363-4a93-9776-188667afbbd5\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6cdf6550-5363-4a93-9776-188667afbbd5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:04:25.535669903Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"region":"6cdf6550-5363-4a93-9776-188667afbbd5","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6cdf6550-5363-4a93-9776-188667afbbd5\" \"--data\" \"/data/regions/6cdf6550-5363-4a93-9776-188667afbbd5\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:04:25.537621361Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"region":"6cdf6550-5363-4a93-9776-188667afbbd5","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:04:25.663724205Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"region":"6cdf6550-5363-4a93-9776-188667afbbd5","component":"worker"}
{"msg":"region 6cdf6550-5363-4a93-9776-188667afbbd5 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:04:25.663783216Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:04:25.663876728Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-6cdf6550-5363-4a93-9776-188667afbbd5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:04:25.664294234Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6cdf6550-5363-4a93-9776-188667afbbd5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:04:25.672660073Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:04:25.689754786Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6cdf6550-5363-4a93-9776-188667afbbd5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:04:25.692376903Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6cdf6550-5363-4a93-9776-188667afbbd5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:04:25.692929531Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:04:25.697027679Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:04:25.69707603Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:04:25.697119761Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:04:25.703780185Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:04:25.71112933Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:04:25.777631226Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"uri":"/crucible/0/regions","method":"POST","req_id":"dec26cd9-1fb0-4ae3-b5e5-d4571671d014","remote_addr":"[fd00:1122:3344:101::a]:54187","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":642,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.881222209Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:35882"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.883553582Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"uri":"/crucible/0/regions/6cdf6550-5363-4a93-9776-188667afbbd5","method":"GET","req_id":"38e8139f-d9c5-4991-a3e4-c00bc9060a60","remote_addr":"[fd00:1122:3344:101::a]:35882","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":2302,"response_code":"200"}
{"msg":"path is \"/data/regions/6cdf6550-5363-4a93-9776-188667afbbd5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.885900085Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"datafile"}
{"msg":"dataset is oxp_18924fa5-0339-4445-8677-913f8ef37483/crucible/regions/6cdf6550-5363-4a93-9776-188667afbbd5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.895027785Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"datafile"}
{"msg":"snapshot is oxp_18924fa5-0339-4445-8677-913f8ef37483/crucible/regions/6cdf6550-5363-4a93-9776-188667afbbd5@8ff1b748-9eef-44e1-9536-0ebb429402dc","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.905408803Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_18924fa5-0339-4445-8677-913f8ef37483/crucible/regions/6cdf6550-5363-4a93-9776-188667afbbd5\", \"8ff1b748-9eef-44e1-9536-0ebb429402dc\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.905443753Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.916775255Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"uri":"/crucible/0/regions/6cdf6550-5363-4a93-9776-188667afbbd5/snapshots/8ff1b748-9eef-44e1-9536-0ebb429402dc","method":"GET","req_id":"322f299c-cf4e-4598-88d0-348196b519f1","remote_addr":"[fd00:1122:3344:101::a]:35882","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":33048,"response_code":"200"}
{"msg":"path is \"/data/regions/6cdf6550-5363-4a93-9776-188667afbbd5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.917645477Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"datafile"}
{"msg":"dataset is oxp_18924fa5-0339-4445-8677-913f8ef37483/crucible/regions/6cdf6550-5363-4a93-9776-188667afbbd5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.926551254Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"datafile"}
{"msg":"snapshot is oxp_18924fa5-0339-4445-8677-913f8ef37483/crucible/regions/6cdf6550-5363-4a93-9776-188667afbbd5@8ff1b748-9eef-44e1-9536-0ebb429402dc","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.93474228Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_18924fa5-0339-4445-8677-913f8ef37483/crucible/regions/6cdf6550-5363-4a93-9776-188667afbbd5\", \"8ff1b748-9eef-44e1-9536-0ebb429402dc\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.93476924Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 6cdf6550-5363-4a93-9776-188667afbbd5-8ff1b748-9eef-44e1-9536-0ebb429402dc state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.943564636Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"datafile"}
{"msg":"applying SMF actions for region 6cdf6550-5363-4a93-9776-188667afbbd5 running snapshot 8ff1b748-9eef-44e1-9536-0ebb429402dc (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.944187405Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.944228145Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"uri":"/crucible/0/regions/6cdf6550-5363-4a93-9776-188667afbbd5/snapshots/8ff1b748-9eef-44e1-9536-0ebb429402dc/run","method":"POST","req_id":"dc51d53b-0515-42da-a4c7-2a7877529cab","remote_addr":"[fd00:1122:3344:101::a]:35882","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":27302,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/6cdf6550-5363-4a93-9776-188667afbbd5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.951963185Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.952054086Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.952140578Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.952148218Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.952191128Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6cdf6550-5363-4a93-9776-188667afbbd5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.952238029Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6cdf6550-5363-4a93-9776-188667afbbd5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.95232908Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.952391891Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.952439772Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.952490193Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.959690245Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-6cdf6550-5363-4a93-9776-188667afbbd5-8ff1b748-9eef-44e1-9536-0ebb429402dc","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.960141001Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-6cdf6550-5363-4a93-9776-188667afbbd5-8ff1b748-9eef-44e1-9536-0ebb429402dc","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.966196458Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-6cdf6550-5363-4a93-9776-188667afbbd5-8ff1b748-9eef-44e1-9536-0ebb429402dc","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.977373487Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6cdf6550-5363-4a93-9776-188667afbbd5/.zfs/snapshot/8ff1b748-9eef-44e1-9536-0ebb429402dc","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.977699291Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.977766112Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.977807283Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.977844173Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.977893614Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.981333483Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.985646704Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"region 6cdf6550-5363-4a93-9776-188667afbbd5 running snapshot 8ff1b748-9eef-44e1-9536-0ebb429402dc state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:26.985660305Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:29.945814347Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:42354"}
{"msg":"region 6af44937-d40d-436d-8294-2377390fa49e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:29.945881658Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:29.948226432Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"uri":"/crucible/0/regions","method":"POST","req_id":"cfd94ac6-2813-4132-9cec-120b62cd9f2a","remote_addr":"[fd00:1122:3344:101::a]:42354","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":2377,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:29.948244142Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_18924fa5-0339-4445-8677-913f8ef37483/crucible/regions/6af44937-d40d-436d-8294-2377390fa49e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:29.957815828Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_18924fa5-0339-4445-8677-913f8ef37483/crucible/regions/6af44937-d40d-436d-8294-2377390fa49e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:29.957838738Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"6af44937-d40d-436d-8294-2377390fa49e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6af44937-d40d-436d-8294-2377390fa49e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.020688672Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"region":"6af44937-d40d-436d-8294-2377390fa49e","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6af44937-d40d-436d-8294-2377390fa49e\" \"--data\" \"/data/regions/6af44937-d40d-436d-8294-2377390fa49e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.020743803Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"region":"6af44937-d40d-436d-8294-2377390fa49e","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.112169834Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"region":"6af44937-d40d-436d-8294-2377390fa49e","component":"worker"}
{"msg":"region 6af44937-d40d-436d-8294-2377390fa49e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.112219005Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.112305866Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-6af44937-d40d-436d-8294-2377390fa49e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.112740333Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6af44937-d40d-436d-8294-2377390fa49e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.115589263Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.119376947Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"uri":"/crucible/0/regions","method":"POST","req_id":"17593918-fc95-43d3-a60e-8094cd21832e","remote_addr":"[fd00:1122:3344:101::a]:42354","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":713,"response_code":"200"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.122016974Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6af44937-d40d-436d-8294-2377390fa49e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.12451906Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6af44937-d40d-436d-8294-2377390fa49e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.124827774Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.124873825Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.124924995Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.124971206Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.128496486Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/6cdf6550-5363-4a93-9776-188667afbbd5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.134281099Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.1343935Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.134486961Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.134494911Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.134537682Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6cdf6550-5363-4a93-9776-188667afbbd5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.134585343Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6cdf6550-5363-4a93-9776-188667afbbd5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.139513643Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.139573674Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.139624784Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.139681485Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.142908231Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:30.144896539Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:54.259228732Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"uri":"/crucible/0/regions/6af44937-d40d-436d-8294-2377390fa49e","method":"GET","req_id":"c2267d36-ecdb-44d8-84bf-0653f994c32a","remote_addr":"[fd00:1122:3344:101::a]:42354","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":56,"response_code":"200"}
{"msg":"path is \"/data/regions/6af44937-d40d-436d-8294-2377390fa49e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:54.26191981Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"datafile"}
{"msg":"dataset is oxp_18924fa5-0339-4445-8677-913f8ef37483/crucible/regions/6af44937-d40d-436d-8294-2377390fa49e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:54.273729148Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"datafile"}
{"msg":"region 6af44937-d40d-436d-8294-2377390fa49e state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:54.281052152Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:54.281259345Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:54.281316946Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"uri":"/crucible/0/regions/6af44937-d40d-436d-8294-2377390fa49e","method":"DELETE","req_id":"56e591de-bedd-4ace-b234-281f9027a245","remote_addr":"[fd00:1122:3344:101::a]:42354","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":19420,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-6af44937-d40d-436d-8294-2377390fa49e (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:54.282363741Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:54.283974124Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"uri":"/crucible/0/regions/6af44937-d40d-436d-8294-2377390fa49e","method":"GET","req_id":"024946dc-5312-40db-80dc-f2364bad2515","remote_addr":"[fd00:1122:3344:101::a]:42354","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":25,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/6cdf6550-5363-4a93-9776-188667afbbd5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:54.2865097Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:54.286591361Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:54.286672492Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:54.286681663Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:54.286721773Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6cdf6550-5363-4a93-9776-188667afbbd5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:54.286767744Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6cdf6550-5363-4a93-9776-188667afbbd5","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:54.287089528Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:54.28716799Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:54.28722106Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:54.287266241Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:54.29073502Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:54.292171151Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_18924fa5-0339-4445-8677-913f8ef37483/crucible/regions/6af44937-d40d-436d-8294-2377390fa49e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:54.301757337Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"region":"6af44937-d40d-436d-8294-2377390fa49e","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:54.376159966Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"uri":"/crucible/0/regions/6af44937-d40d-436d-8294-2377390fa49e","method":"GET","req_id":"450cb98e-8b8e-4306-94fd-9e0ee1611530","remote_addr":"[fd00:1122:3344:101::a]:42354","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":694,"response_code":"200"}
{"msg":"region 6af44937-d40d-436d-8294-2377390fa49e state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:54.381027845Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:09:54.44811227Z","hostname":"oxz_crucible_d0587fa8-7f4f-4760-8e06-f0b99f84ac79","pid":25241,"uri":"/crucible/0/regions/6af44937-d40d-436d-8294-2377390fa49e","method":"GET","req_id":"861e4363-9957-4c5b-acc1-d546cc027c18","remote_addr":"[fd00:1122:3344:101::a]:42354","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":41,"response_code":"200"}